experiment: test deterministic user source identity

2026-08-07 15:24:47 +01:00
parent d2d84656fe
commit 3537aa1b7a
3 changed files with 243 additions and 6 deletions
@@ -0,0 +1,106 @@
+import { describe, it, expect } from "vitest";
+import { createHash } from "node:crypto";
+
+/**
+ * TEST-ONLY helper. Not exported into production.
+ *
+ * Creates a deterministic source record from raw user input.
+ * This exists only to test whether trustworthy source identity
+ * can be established before any LLM interpretation occurs.
+ */
+function createSourceRecord(rawInput) {
+  // Defensive clone so the caller's original string is never mutated
+  const verbatimText = String(rawInput);
+
+  const record = {
+    sourceId: createHash("sha256")
+      .update(verbatimText, "utf8")
+      .digest("hex"),
+    sourceType: "user_input",
+    verbatimText,
+  };
+
+  return record;
+}
+
+// ──────────────────────────────────────────────
+// Four fixed inputs (Experiment 54H)
+// ──────────────────────────────────────────────
+
+const CASE_1 = "The business has lost three major customers this year.";
+const CASE_2 = "Three major customers have left the business this year.";
+const CASE_3 = "Revenue is down. I think pricing may be part of the problem, but I am not sure.";
+const CASE_4 = "The business has lost three major customers this year."; // identical to Case 1
+
+describe("Deterministic source record (Exp 54H — test-only)", () => {
+  it("Case 1: verbatimText equals the exact input string", () => {
+    const record = createSourceRecord(CASE_1);
+    expect(record.verbatimText).toBe(CASE_1);
+  });
+
+  it("Case 2 (paraphrase): sourceType is always user_input", () => {
+    const record2 = createSourceRecord(CASE_2);
+    const record3 = createSourceRecord(CASE_3);
+    expect(record2.sourceType).toBe("user_input");
+    expect(record3.sourceType).toBe("user_input");
+  });
+
+  it("Case 1 and Case 4 produce identical sourceId", () => {
+    const r1 = createSourceRecord(CASE_1);
+    const r4 = createSourceRecord(CASE_4);
+    expect(r1.sourceId).toBe(r4.sourceId);
+  });
+
+  it("Case 1 and Case 2 produce different sourceIds despite similar meaning", () => {
+    const r1 = createSourceRecord(CASE_1);
+    const r2 = createSourceRecord(CASE_2);
+    expect(r1.sourceId).not.toBe(r2.sourceId);
+  });
+
+  it("Case 3: multi-sentence verbatim text is preserved unchanged", () => {
+    const record = createSourceRecord(CASE_3);
+    expect(record.verbatimText).toBe(CASE_3);
+  });
+
+  it("Input string is not mutated by createSourceRecord", () => {
+    let originalLength = CASE_1.length;
+    createSourceRecord(CASE_1);
+    expect(CASE_1.length).toBe(originalLength);
+    // Also verify no characters were altered (strings are immutable in JS,
+    // but be explicit about the contract)
+    const r = createSourceRecord(CASE_1);
+    expect(r.verbatimText).toBe(CASE_1);
+  });
+
+  it("Helper output is deterministic across repeated calls", () => {
+    const rA = createSourceRecord(CASE_1);
+    const rB = createSourceRecord(CASE_1);
+    const rC = createSourceRecord(CASE_2);
+    expect(rA).toEqual(rB);
+    expect(rA.sourceId).not.toBe(rC.sourceId);
+  });
+
+  it("No interpretation or summarisation occurs in the source record", () => {
+    // The record must contain only source identity fields, never
+    // evidence type, confidence, relevance, summary, etc.
+    const record = createSourceRecord(CASE_3);
+    const keys = Object.keys(record);
+    expect(keys).toEqual(expect.arrayContaining(["sourceId", "sourceType", "verbatimText"]));
+    expect(keys.length).toBe(3);
+    // No extra fields that would imply interpretation
+    expect(keys).not.toContain("evidenceType");
+    expect(keys).not.toContain("confidence");
+    expect(keys).not.toContain("relevance");
+    expect(keys).not.toContain("summary");
+    expect(keys).not.toContain("interpretation");
+  });
+
+  it("Different inputs produce different IDs (distinct text)", () => {
+    const r1 = createSourceRecord(CASE_1);
+    const r2 = createSourceRecord(CASE_2);
+    const r3 = createSourceRecord(CASE_3);
+    expect(r1.sourceId).not.toBe(r2.sourceId);
+    expect(r1.sourceId).not.toBe(r3.sourceId);
+    expect(r2.sourceId).not.toBe(r3.sourceId);
+  });
+});