experiment: separate source support from interpretation additions

2026-08-07 16:12:29 +01:00
parent c2debad66d
commit 0f7457d9c1
3 changed files with 494 additions and 4 deletions
@@ -0,0 +1,297 @@
+import { describe, it, expect } from "vitest";
+import { createHash } from "node:crypto";
+
+/**
+ * TEST-ONLY helpers for Experiment 54J.
+ * NOT exported into production.
+ */
+
+function createSourceRecord(rawInput) {
+  const verbatimText = String(rawInput);
+  return {
+    sourceId: createHash("sha256")
+      .update(verbatimText, "utf8")
+      .digest("hex"),
+    sourceType: "user_input",
+    verbatimText,
+  };
+}
+
+function createInterpretationRecord(sourceId, interpretationText) {
+  const combined = sourceId + "|" + interpretationText;
+  return {
+    interpretationId: createHash("sha256")
+      .update(combined, "utf8")
+      .digest("hex"),
+    sourceId,
+    interpretationText,
+  };
+}
+
+/**
+ * Test-only grounding record shape for Experiment 54J.
+ * NOT a proposed production schema.
+ * Does not include: score, confidence, status, relevance, correctness, next-question fields.
+ */
+function createGroundingRecord(sourceId, interpretationId, supportedStatements, addedStatements) {
+  return {
+    sourceId,
+    interpretationId,
+    supportedBySource: [...supportedStatements],
+    addedByInterpretation: [...addedStatements],
+  };
+}
+
+// ──────────────────────────────────────────────
+// Fixed inputs for Experiment 54J
+// ──────────────────────────────────────────────
+
+const RAW_USER_INPUT =
+  "Revenue is down. I think pricing may be part of the problem, but I am not sure.";
+
+const INTERPRETATION_A_TEXT =
+  "Pricing may be contributing materially to the revenue decline.";
+
+const INTERPRETATION_B_TEXT =
+  "The revenue decline may have causes other than pricing, and pricing has not yet been established as the main problem.";
+
+// ──────────────────────────────────────────────
+// Human-fixed grounding references (pre-written test data)
+// These are NOT generated dynamically.
+// They represent the human-reviewed answer to:
+//   "what is supported by source vs added by interpretation?"
+// ──────────────────────────────────────────────
+
+const SUPPORTED_BY_SOURCE_A = [
+  "revenue is down",
+  "pricing may be part of the problem",
+];
+
+const ADDED_BY_INTERPRETATION_A = [
+  "pricing may be contributing materially to the decline",
+];
+
+const SUPPORTED_BY_SOURCE_B = [
+  "revenue is down",
+  "pricing may be part of the problem",
+  "the user is unsure",
+];
+
+const ADDED_BY_INTERPRETATION_B = [
+  "there may be causes other than pricing",
+  "pricing has not yet been established as the main problem",
+];
+
+
+describe("Interpretation-source grounding comparison (Exp 54J - test-only)", () => {
+  let source;
+  let interpretationA;
+  let interpretationB;
+  let groundingA;
+  let groundingB;
+
+  beforeEach(() => {
+    const record = createSourceRecord(RAW_USER_INPUT);
+    source = record;
+    interpretationA = createInterpretationRecord(source.sourceId, INTERPRETATION_A_TEXT);
+    interpretationB = createInterpretationRecord(source.sourceId, INTERPRETATION_B_TEXT);
+
+    groundingA = createGroundingRecord(
+      source.sourceId,
+      interpretationA.interpretationId,
+      SUPPORTED_BY_SOURCE_A,
+      ADDED_BY_INTERPRETATION_A,
+    );
+
+    groundingB = createGroundingRecord(
+      source.sourceId,
+      interpretationB.interpretationId,
+      SUPPORTED_BY_SOURCE_B,
+      ADDED_BY_INTERPRETATION_B,
+    );
+  });
+
+  // ── Test 1: Both grounding records reference the same exact sourceId ──
+  it("Test 1: both grounding records share the same sourceId", () => {
+    expect(groundingA.sourceId).toBe(source.sourceId);
+    expect(groundingB.sourceId).toBe(source.sourceId);
+    expect(groundingA.sourceId).toBe(groundingB.sourceId);
+  });
+
+  // ── Test 2: Each grounding record references the correct distinct interpretationId ──
+  it("Test 2: each grounding record references its distinct interpretationId", () => {
+    expect(groundingA.interpretationId).toBe(interpretationA.interpretationId);
+    expect(groundingB.interpretationId).toBe(interpretationB.interpretationId);
+    expect(groundingA.interpretationId).not.toBe(groundingB.interpretationId);
+  });
+
+  // ── Test 3: source-supported statements are separate from interpretation-added for each record ──
+  it("Test 3: supported vs added remain separate within each grounding record", () => {
+    const supportedSetA = new Set(groundingA.supportedBySource);
+    const addedSetA = new Set(groundingA.addedByInterpretation);
+    for (const stmt of groundingA.supportedBySource) {
+      expect(addedSetA.has(stmt)).toBe(false);
+    }
+
+    const supportedSetB = new Set(groundingB.supportedBySource);
+    const addedSetB = new Set(groundingB.addedByInterpretation);
+    for (const stmt of groundingB.supportedBySource) {
+      expect(addedSetB.has(stmt)).toBe(false);
+    }
+  });
+
+  // ── Test 4: Interpretation A records 'materially' as added meaning ──
+  it("Test 4: interpretation A records 'materially' as added meaning", () => {
+    const hasMaterially = groundingA.addedByInterpretation.some(
+      (s) => s.toLowerCase().includes("materially"),
+    );
+    expect(hasMaterially).toBe(true);
+  });
+
+  // ── Test 5: Interpretation B records alternative causes as added meaning ──
+  it("Test 5: interpretation B records 'alternative causes' as added meaning", () => {
+    const hasAlternativeCause = groundingB.addedByInterpretation.some(
+      (s) => s.toLowerCase().includes("other than pricing"),
+    );
+    expect(hasAlternativeCause).toBe(true);
+  });
+
+  // ── Test 6: B records "not established as the main problem" as interpretation-added ──
+  it("Test 6: B's 'not established as main problem' is interpretation-added, not user-stated fact", () => {
+    const hasNotEstablished = groundingB.addedByInterpretation.some(
+      (s) => s.toLowerCase().includes("established"),
+    );
+    expect(hasNotEstablished).toBe(true);
+
+    // It should NOT appear in supportedBySource
+    const inSupported = groundingB.supportedBySource.some(
+      (s) => s.toLowerCase().includes("main problem") || s.toLowerCase().includes("established"),
+    );
+    expect(inSupported).toBe(false);
+  });
+
+  // ── Test 7: Neither grounding record mutates the source ──
+  it("Test 7: neither grounding record mutates the source", () => {
+    const verbatimBefore = source.verbatimText;
+    const sourceIdBefore = source.sourceId;
+    const keysBefore = Object.keys(source);
+
+    expect(groundingA.sourceId).toBe(sourceIdBefore);
+    expect(groundingB.sourceId).toBe(sourceIdBefore);
+
+    expect(source.verbatimText).toBe(verbatimBefore);
+    expect(source.sourceId).toBe(sourceIdBefore);
+    expect(Object.keys(source)).toEqual(keysBefore);
+  });
+
+  // ── Test 8: Neither grounding record mutates the interpretation records ──
+  it("Test 8: grounding creation does not affect interpretationId values", () => {
+    const aIdBefore = interpretationA.interpretationId;
+    const bIdBefore = interpretationB.interpretationId;
+
+    // Create fresh groundings in parallel
+    const gA2 = createGroundingRecord(
+      source.sourceId,
+      interpretationA.interpretationId,
+      [...SUPPORTED_BY_SOURCE_A],
+      [...ADDED_BY_INTERPRETATION_A],
+    );
+    const gB2 = createGroundingRecord(
+      source.sourceId,
+      interpretationB.interpretationId,
+      [...SUPPORTED_BY_SOURCE_B],
+      [...ADDED_BY_INTERPRETATION_B],
+    );
+
+    expect(interpretationA.interpretationId).toBe(aIdBefore);
+    expect(interpretationB.interpretationId).toBe(bIdBefore);
+    expect(groundingA.interpretationId).toBe(gA2.interpretationId);
+    expect(groundingB.interpretationId).toBe(gB2.interpretationId);
+  });
+
+  // ── Test 9: Recreating the same grounding record is deterministic ──
+  it("Test 9: recreating the same grounding record produces identical results", () => {
+    const gA_recreate = createGroundingRecord(
+      source.sourceId,
+      interpretationA.interpretationId,
+      [...SUPPORTED_BY_SOURCE_A],
+      [...ADDED_BY_INTERPRETATION_A],
+    );
+
+    const gB_recreate = createGroundingRecord(
+      source.sourceId,
+      interpretationB.interpretationId,
+      [...SUPPORTED_BY_SOURCE_B],
+      [...ADDED_BY_INTERPRETATION_B],
+    );
+
+    expect(gA_recreate.sourceId).toBe(groundingA.sourceId);
+    expect(gA_recreate.interpretationId).toBe(groundingA.interpretationId);
+    expect(gA_recreate.supportedBySource).toEqual(groundingA.supportedBySource);
+    expect(gA_recreate.addedByInterpretation).toEqual(groundingA.addedByInterpretation);
+
+    expect(gB_recreate.sourceId).toBe(groundingB.sourceId);
+    expect(gB_recreate.interpretationId).toBe(groundingB.interpretationId);
+    expect(gB_recreate.supportedBySource).toEqual(groundingB.supportedBySource);
+    expect(gB_recreate.addedByInterpretation).toEqual(groundingB.addedByInterpretation);
+  });
+
+  // ── Test 10: Later consumer can inspect A and B to distinguish shared source, distinct interpretations, supported vs added ──
+  it("Test 10: a later consumer can inspect A and B and distinguish shared source / distinct interpretation / supported vs added", () => {
+    expect(groundingA.sourceId).toBe(groundingB.sourceId);
+
+    expect(groundingA.interpretationId).not.toBe(groundingB.interpretationId);
+
+    const consumerView = {
+      sharedSourceId: groundingA.sourceId,
+      interpretationA: {
+        id: groundingA.interpretationId,
+        supportedBySource: [...groundingA.supportedBySource],
+        addedByInterpretation: [...groundingA.addedByInterpretation],
+      },
+      interpretationB: {
+        id: groundingB.interpretationId,
+        supportedBySource: [...groundingB.supportedBySource],
+        addedByInterpretation: [...groundingB.addedByInterpretation],
+      },
+    };
+
+    expect(consumerView.sharedSourceId).toBeDefined();
+    expect(consumerView.interpretationA.id).not.toBe(consumerView.interpretationB.id);
+    expect(consumerView.interpretationA.supportedBySource.length).toBeGreaterThan(0);
+    expect(consumerView.interpretationA.addedByInterpretation.length).toBeGreaterThan(0);
+    expect(consumerView.interpretationB.supportedBySource.length).toBeGreaterThan(0);
+    expect(consumerView.interpretationB.addedByInterpretation.length).toBeGreaterThan(0);
+  });
+
+  // ── Test 11: No winner is selected ──
+  it("Test 11: neither interpretation is selected as a winner", () => {
+    const groundingKeysA = Object.keys(groundingA);
+    const groundingKeysB = Object.keys(groundingB);
+
+    expect(groundingKeysA).not.toContain("correctness");
+    expect(groundingKeysA).not.toContain("status");
+    expect(groundingKeysB).not.toContain("correctness");
+    expect(groundingKeysB).not.toContain("status");
+    expect(groundingKeysB).toEqual(groundingKeysA);
+  });
+
+  // ── Test 12: No numeric comparison is performed ──
+  it("Test 12: no numeric scoring is present in grounding records", () => {
+    const allStrings = [
+      ...groundingA.supportedBySource,
+      ...groundingA.addedByInterpretation,
+      ...groundingB.supportedBySource,
+      ...groundingB.addedByInterpretation,
+    ];
+
+    for (const stmt of allStrings) {
+      expect(typeof stmt).toBe("string");
+    }
+  });
+
+  // ── Test 13: No LLM/network call occurs ──
+  it("Test 13: no LLM or network call occurs", () => {
+    expect(true).toBe(true);
+  });
+});