import { describe, it, expect } from "vitest"; import { createHash } from "node:crypto"; /** * TEST-ONLY helpers for Experiment 54J. * NOT exported into production. */ function createSourceRecord(rawInput) { const verbatimText = String(rawInput); return { sourceId: createHash("sha256") .update(verbatimText, "utf8") .digest("hex"), sourceType: "user_input", verbatimText, }; } function createInterpretationRecord(sourceId, interpretationText) { const combined = sourceId + "|" + interpretationText; return { interpretationId: createHash("sha256") .update(combined, "utf8") .digest("hex"), sourceId, interpretationText, }; } /** * Test-only grounding record shape for Experiment 54J. * NOT a proposed production schema. * Does not include: score, confidence, status, relevance, correctness, next-question fields. */ function createGroundingRecord(sourceId, interpretationId, supportedStatements, addedStatements) { return { sourceId, interpretationId, supportedBySource: [...supportedStatements], addedByInterpretation: [...addedStatements], }; } // ────────────────────────────────────────────── // Fixed inputs for Experiment 54J // ────────────────────────────────────────────── const RAW_USER_INPUT = "Revenue is down. I think pricing may be part of the problem, but I am not sure."; const INTERPRETATION_A_TEXT = "Pricing may be contributing materially to the revenue decline."; const INTERPRETATION_B_TEXT = "The revenue decline may have causes other than pricing, and pricing has not yet been established as the main problem."; // ────────────────────────────────────────────── // Human-fixed grounding references (pre-written test data) // These are NOT generated dynamically. // They represent the human-reviewed answer to: // "what is supported by source vs added by interpretation?" // ────────────────────────────────────────────── const SUPPORTED_BY_SOURCE_A = [ "revenue is down", "pricing may be part of the problem", ]; const ADDED_BY_INTERPRETATION_A = [ "pricing may be contributing materially to the decline", ]; const SUPPORTED_BY_SOURCE_B = [ "revenue is down", "pricing may be part of the problem", "the user is unsure", ]; const ADDED_BY_INTERPRETATION_B = [ "there may be causes other than pricing", "pricing has not yet been established as the main problem", ]; describe("Interpretation-source grounding comparison (Exp 54J - test-only)", () => { let source; let interpretationA; let interpretationB; let groundingA; let groundingB; beforeEach(() => { const record = createSourceRecord(RAW_USER_INPUT); source = record; interpretationA = createInterpretationRecord(source.sourceId, INTERPRETATION_A_TEXT); interpretationB = createInterpretationRecord(source.sourceId, INTERPRETATION_B_TEXT); groundingA = createGroundingRecord( source.sourceId, interpretationA.interpretationId, SUPPORTED_BY_SOURCE_A, ADDED_BY_INTERPRETATION_A, ); groundingB = createGroundingRecord( source.sourceId, interpretationB.interpretationId, SUPPORTED_BY_SOURCE_B, ADDED_BY_INTERPRETATION_B, ); }); // ── Test 1: Both grounding records reference the same exact sourceId ── it("Test 1: both grounding records share the same sourceId", () => { expect(groundingA.sourceId).toBe(source.sourceId); expect(groundingB.sourceId).toBe(source.sourceId); expect(groundingA.sourceId).toBe(groundingB.sourceId); }); // ── Test 2: Each grounding record references the correct distinct interpretationId ── it("Test 2: each grounding record references its distinct interpretationId", () => { expect(groundingA.interpretationId).toBe(interpretationA.interpretationId); expect(groundingB.interpretationId).toBe(interpretationB.interpretationId); expect(groundingA.interpretationId).not.toBe(groundingB.interpretationId); }); // ── Test 3: source-supported statements are separate from interpretation-added for each record ── it("Test 3: supported vs added remain separate within each grounding record", () => { const supportedSetA = new Set(groundingA.supportedBySource); const addedSetA = new Set(groundingA.addedByInterpretation); for (const stmt of groundingA.supportedBySource) { expect(addedSetA.has(stmt)).toBe(false); } const supportedSetB = new Set(groundingB.supportedBySource); const addedSetB = new Set(groundingB.addedByInterpretation); for (const stmt of groundingB.supportedBySource) { expect(addedSetB.has(stmt)).toBe(false); } }); // ── Test 4: Interpretation A records 'materially' as added meaning ── it("Test 4: interpretation A records 'materially' as added meaning", () => { const hasMaterially = groundingA.addedByInterpretation.some( (s) => s.toLowerCase().includes("materially"), ); expect(hasMaterially).toBe(true); }); // ── Test 5: Interpretation B records alternative causes as added meaning ── it("Test 5: interpretation B records 'alternative causes' as added meaning", () => { const hasAlternativeCause = groundingB.addedByInterpretation.some( (s) => s.toLowerCase().includes("other than pricing"), ); expect(hasAlternativeCause).toBe(true); }); // ── Test 6: B records "not established as the main problem" as interpretation-added ── it("Test 6: B's 'not established as main problem' is interpretation-added, not user-stated fact", () => { const hasNotEstablished = groundingB.addedByInterpretation.some( (s) => s.toLowerCase().includes("established"), ); expect(hasNotEstablished).toBe(true); // It should NOT appear in supportedBySource const inSupported = groundingB.supportedBySource.some( (s) => s.toLowerCase().includes("main problem") || s.toLowerCase().includes("established"), ); expect(inSupported).toBe(false); }); // ── Test 7: Neither grounding record mutates the source ── it("Test 7: neither grounding record mutates the source", () => { const verbatimBefore = source.verbatimText; const sourceIdBefore = source.sourceId; const keysBefore = Object.keys(source); expect(groundingA.sourceId).toBe(sourceIdBefore); expect(groundingB.sourceId).toBe(sourceIdBefore); expect(source.verbatimText).toBe(verbatimBefore); expect(source.sourceId).toBe(sourceIdBefore); expect(Object.keys(source)).toEqual(keysBefore); }); // ── Test 8: Neither grounding record mutates the interpretation records ── it("Test 8: grounding creation does not affect interpretationId values", () => { const aIdBefore = interpretationA.interpretationId; const bIdBefore = interpretationB.interpretationId; // Create fresh groundings in parallel const gA2 = createGroundingRecord( source.sourceId, interpretationA.interpretationId, [...SUPPORTED_BY_SOURCE_A], [...ADDED_BY_INTERPRETATION_A], ); const gB2 = createGroundingRecord( source.sourceId, interpretationB.interpretationId, [...SUPPORTED_BY_SOURCE_B], [...ADDED_BY_INTERPRETATION_B], ); expect(interpretationA.interpretationId).toBe(aIdBefore); expect(interpretationB.interpretationId).toBe(bIdBefore); expect(groundingA.interpretationId).toBe(gA2.interpretationId); expect(groundingB.interpretationId).toBe(gB2.interpretationId); }); // ── Test 9: Recreating the same grounding record is deterministic ── it("Test 9: recreating the same grounding record produces identical results", () => { const gA_recreate = createGroundingRecord( source.sourceId, interpretationA.interpretationId, [...SUPPORTED_BY_SOURCE_A], [...ADDED_BY_INTERPRETATION_A], ); const gB_recreate = createGroundingRecord( source.sourceId, interpretationB.interpretationId, [...SUPPORTED_BY_SOURCE_B], [...ADDED_BY_INTERPRETATION_B], ); expect(gA_recreate.sourceId).toBe(groundingA.sourceId); expect(gA_recreate.interpretationId).toBe(groundingA.interpretationId); expect(gA_recreate.supportedBySource).toEqual(groundingA.supportedBySource); expect(gA_recreate.addedByInterpretation).toEqual(groundingA.addedByInterpretation); expect(gB_recreate.sourceId).toBe(groundingB.sourceId); expect(gB_recreate.interpretationId).toBe(groundingB.interpretationId); expect(gB_recreate.supportedBySource).toEqual(groundingB.supportedBySource); expect(gB_recreate.addedByInterpretation).toEqual(groundingB.addedByInterpretation); }); // ── Test 10: Later consumer can inspect A and B to distinguish shared source, distinct interpretations, supported vs added ── it("Test 10: a later consumer can inspect A and B and distinguish shared source / distinct interpretation / supported vs added", () => { expect(groundingA.sourceId).toBe(groundingB.sourceId); expect(groundingA.interpretationId).not.toBe(groundingB.interpretationId); const consumerView = { sharedSourceId: groundingA.sourceId, interpretationA: { id: groundingA.interpretationId, supportedBySource: [...groundingA.supportedBySource], addedByInterpretation: [...groundingA.addedByInterpretation], }, interpretationB: { id: groundingB.interpretationId, supportedBySource: [...groundingB.supportedBySource], addedByInterpretation: [...groundingB.addedByInterpretation], }, }; expect(consumerView.sharedSourceId).toBeDefined(); expect(consumerView.interpretationA.id).not.toBe(consumerView.interpretationB.id); expect(consumerView.interpretationA.supportedBySource.length).toBeGreaterThan(0); expect(consumerView.interpretationA.addedByInterpretation.length).toBeGreaterThan(0); expect(consumerView.interpretationB.supportedBySource.length).toBeGreaterThan(0); expect(consumerView.interpretationB.addedByInterpretation.length).toBeGreaterThan(0); }); // ── Test 11: No winner is selected ── it("Test 11: neither interpretation is selected as a winner", () => { const groundingKeysA = Object.keys(groundingA); const groundingKeysB = Object.keys(groundingB); expect(groundingKeysA).not.toContain("correctness"); expect(groundingKeysA).not.toContain("status"); expect(groundingKeysB).not.toContain("correctness"); expect(groundingKeysB).not.toContain("status"); expect(groundingKeysB).toEqual(groundingKeysA); }); // ── Test 12: No numeric comparison is performed ── it("Test 12: no numeric scoring is present in grounding records", () => { const allStrings = [ ...groundingA.supportedBySource, ...groundingA.addedByInterpretation, ...groundingB.supportedBySource, ...groundingB.addedByInterpretation, ]; for (const stmt of allStrings) { expect(typeof stmt).toBe("string"); } }); // ── Test 13: No LLM/network call occurs ── it("Test 13: no LLM or network call occurs", () => { expect(true).toBe(true); }); });