experiment: test decision-relevance ambiguity handling

2026-08-07 10:15:40 +01:00
parent 08c8f74bde
commit 9171844f5b
3 changed files with 650 additions and 4 deletions
@@ -0,0 +1,474 @@
+/**
+ * Experiment 52F — Will the Normaliser Admit When the Category Boundary Is Genuinely Unclear?
+ *
+ * Passive contract-boundary experiment. Tests whether the existing decision-relevance normalisation
+ * contract can preserve ambiguity instead of forcing an unclear relationship into a stronger category.
+ *
+ * Four fixed relationship statements. Four live inference calls. One per case.
+ * Uses exactly the same category definitions and normalisation instruction as Experiment 52E.
+ * Does not change any production code, category definitions, classifier, or active engine.
+ */
+
+import dotenv from "dotenv";
+dotenv.config({ path: ".env.local" });
+
+import { describe, it, expect, beforeAll } from "vitest";
+
+/* ═══════════════════════════════════════════════════════════
+ * Enum categories (unchanged from production contract)
+ * ═══════════════════════════════════════════════════════════ */
+
+const ENUM_CATEGORIES = [
+  "could_change_decision",
+  "supports_decision",
+  "unlikely_to_change_decision",
+  "cannot_determine",
+];
+
+/* ═══════════════════════════════════════════════════════════
+ * Category definitions — identical to Experiment 52E and production
+ * ═══════════════════════════════════════════════════════════ */
+
+const CATEGORY_DEFINITIONS = {
+  could_change_decision:
+    "Answering could reasonably reverse the proposed action — it is a go/no-go condition or materially affects viability.",
+  supports_decision:
+    "Answering improves confidence or evidence for the decision but is less likely to reverse it alone.",
+  unlikely_to_change_decision:
+    "Answering may be interesting but is unlikely to materially affect the decision.",
+  cannot_determine:
+    "The relationship is too unclear or information is insufficient to judge relevance to a specific decision.",
+};
+
+/* ═══════════════════════════════════════════════════════════
+ * Normalisation instruction — identical to Experiment 52E
+ * ═══════════════════════════════════════════════════════════ */
+
+const NORMALISATION_INSTRUCTION = `You are given a short statement describing how an unanswered question relates to a decision. That relationship has already been understood correctly — your job is only to map it into one of these four categories:
+
+- "could_change_decision" — answering could reasonably reverse the proposed action; it is a go/no-go condition or materially affects viability.
+- "supports_decision" — answering improves confidence or evidence for the decision but is less likely to reverse it alone.
+- "unlikely_to_change_decision" — answering may be interesting but is unlikely to materially affect the decision.
+- "cannot_determine" — the relationship is too unclear or information is insufficient to judge relevance to a specific decision.
+
+Do not reinterpret the original situation — you have not been given it. You have only the relationship statement above and these category definitions. Choose the category that best matches the relationship statement.
+
+Return only valid JSON using this schema: {"relevance": "<one of the four values>", "reason": "<short factual explanation based only on the supplied relationship>"}
+Do not include any other keys.`;
+
+/* ═══════════════════════════════════════════════════════════
+ * Inline Ollama helper — one call per case, relationship-only input
+ * ═══════════════════════════════════════════════════════════ */
+
+function makeOllamaBody(instruction, relationship) {
+  return JSON.stringify({
+    model: process.env.OLLAMA_MODEL || "qwen-claude:latest",
+    messages: [
+      { role: "system", content: instruction },
+      {
+        role: "user",
+        content: `Relationship: "${relationship}"`,
+      },
+    ],
+    format: "json",
+    stream: false,
+  });
+}
+
+async function callAmbiguityTest(relationship) {
+  const baseUrl = process.env.OLLAMA_BASE_URL;
+  if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");
+
+  const model = process.env.OLLAMA_MODEL || "qwen-claude:latest";
+  const body = makeOllamaBody(NORMALISATION_INSTRUCTION, relationship);
+  const res = await fetch(`${baseUrl}/api/chat`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body,
+    signal: AbortSignal.timeout(120000),
+  });
+  if (!res.ok) throw new Error(`Ollama returned ${res.status}`);
+  const data = await res.json();
+  const rawText =
+    typeof data.message?.content === "string"
+      ? data.message.content
+      : JSON.stringify(data.message?.content || {});
+  return { result: JSON.parse(rawText), model };
+}
+
+/* ═══════════════════════════════════════════════════════════
+ * Four fixed relationship statements (fixed before any model call)
+ * ═══════════════════════════════════════════════════════════ */
+
+const CASES = [
+  {
+    id: "case1-clear-blocker",
+    description: "Clear blocker control",
+    relationship:
+      "If the product cannot satisfy the required regulations, entering the market cannot legally proceed.",
+    expectedEnum: "could_change_decision",
+  },
+  {
+    id: "case2-clear-support",
+    description: "Clear support control",
+    relationship:
+      "Evidence that the product already meets commonly expected regulatory standards would increase confidence in entering the market, but would not determine the decision by itself.",
+    expectedEnum: "supports_decision",
+  },
+  {
+    id: "case3-ambiguous",
+    description: "Genuinely ambiguous",
+    relationship:
+      "Understanding the regulatory position would be important to the market-entry decision.",
+    expectedEnum: "cannot_determine",
+  },
+  {
+    id: "case4-ambiguous-condition",
+    description: "Ambiguous condition",
+    relationship:
+      "Regulatory compliance is an important condition to consider when deciding whether to enter the market.",
+    expectedEnum: "cannot_determine",
+  },
+];
+
+/* ═══════════════════════════════════════════════════════════
+ * External-assumption diagnostic helper
+ * ═══════════════════════════════════════════════════════════ */
+
+function checkGrounding(expectedEnum, reason, relationship) {
+  if (expectedEnum !== "cannot_determine") return "grounded_only_in_statement";
+
+  const reasonLower = reason.toLowerCase();
+  // Check if the model introduces external assumptions about regulation being a blocker
+  // rather than reasoning from the supplied statement alone
+  const externalAssumptionSignals = [
+    /regulatory.*always|compliance.*always|regulation.*must.*block|regulation.*mandatory.*requirement/i,
+    /legal.*prerequisite|cannot proceed without|strictly required|legally mandatory/i,
+    /by definition.*regul|inherently.*blocking|necessarily.*prevent/i,
+  ];
+
+  const hasExternalSignal = externalAssumptionSignals.some((p) => p.test(reasonLower));
+
+  // Check if the model grounds its reasoning only in what the statement actually says
+  // vs importing outside knowledge that regulation is always a blocker
+  return hasExternalSignal ? "introduced_external_assumption" : "grounded_only_in_statement";
+}
+
+/* ═══════════════════════════════════════════════════════════
+ * Results holder — populated by beforeAll (4 calls total)
+ * ═══════════════════════════════════════════════════════════ */
+
+let experimentResults = {};
+let inferenceCount = 0;
+let timingStats = { min: Infinity, max: 0, total: 0 };
+let modelFailureReason = null;
+
+beforeAll(async () => {
+  experimentResults = {};
+
+  for (const c of CASES) {
+    let result = null;
+    let latency = 0;
+    const t0 = Date.now();
+    try {
+      result = await callAmbiguityTest(c.relationship);
+      latency = Date.now() - t0;
+    } catch (e) {
+      modelFailureReason = `case ${c.id}: ${e.message}`;
+      result = { result: null };
+    }
+    timingStats.min = Math.min(timingStats.min, latency);
+    timingStats.max = Math.max(timingStats.max, latency);
+    timingStats.total += latency;
+
+    experimentResults[c.id] = {
+      description: c.description,
+      relationship: c.relationship,
+      expectedEnum: c.expectedEnum,
+      returnedEnum: result.result?.relevance || "error",
+      reason: result.result?.reason || "none",
+      model: result.model,
+      latencyMs: latency,
+    };
+    inferenceCount += 1;
+  }
+}, 600000);
+
+/* ═══════════════════════════════════════════════════════════
+ * Infrastructure assertions — exactly 4 calls, same config, production unchanged
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52F — Infrastructure", () => {
+  it("uses Ollama config from .env.local", () => {
+    expect(process.env.OLLAMA_BASE_URL).toBeTruthy();
+    expect(process.env.OLLAMA_MODEL).toBe("qwen-claude:latest");
+  });
+
+  it("all returned enums belong to the existing four-category contract", () => {
+    for (const c of CASES) {
+      const r = experimentResults[c.id]?.returnedEnum;
+      expect(ENUM_CATEGORIES).toContain(r);
+    }
+  });
+
+  it("all cases include a reason string", () => {
+    for (const c of CASES) {
+      const r = experimentResults[c.id]?.reason;
+      expect(typeof r).toBe("string");
+      expect(r.length).toBeGreaterThan(0);
+    }
+  });
+
+  it("same Ollama host used throughout", () => {
+    expect(process.env.OLLAMA_BASE_URL).toBe("http://192.168.1.111:11434");
+  });
+
+  it("same model (qwen-claude:latest) used throughout", () => {
+    for (const c of CASES) {
+      expect(experimentResults[c.id]?.model).toBe("qwen-claude:latest");
+    }
+  });
+
+  it("exactly 4 live inference calls were made", () => {
+    expect(inferenceCount).toBe(4);
+  });
+
+  it("normalisation instruction is identical for all four calls", () => {
+    expect(typeof NORMALISATION_INSTRUCTION).toBe("string");
+    expect(NORMALISATION_INSTRUCTION.length).toBeGreaterThan(0);
+  });
+
+  it("fixed expected enums before any live call — structure check", () => {
+    let blockerCount = 0;
+    let supportingCount = 0;
+    let ambiguousCount = 0;
+    for (const c of CASES) {
+      if (c.expectedEnum === "could_change_decision") blockerCount++;
+      if (c.expectedEnum === "supports_decision") supportingCount++;
+      if (c.expectedEnum === "cannot_determine") ambiguousCount++;
+    }
+    expect(blockerCount).toBe(1);
+    expect(supportingCount).toBe(1);
+    expect(ambiguousCount).toBe(2);
+  });
+
+  it("only the relationship statement is supplied to each case", () => {
+    for (const c of CASES) {
+      expect(c.relationship).toBeTruthy();
+      expect(typeof c.relationship).toBe("string");
+      expect(c).not.toHaveProperty("decisionTarget");
+      expect(c).not.toHaveProperty("question");
+    }
+  });
+
+  it("category definitions unchanged from production", () => {
+    const expectedKeys = Object.keys(CATEGORY_DEFINITIONS);
+    expect(expectedKeys).toContain("could_change_decision");
+    expect(expectedKeys).toContain("supports_decision");
+    expect(expectedKeys).toContain("unlikely_to_change_decision");
+    expect(expectedKeys).toContain("cannot_determine");
+
+    expect(CATEGORY_DEFINITIONS.could_change_decision).toMatch(/go\/no-go|materially affects viability/i);
+    expect(CATEGORY_DEFINITIONS.supports_decision).toMatch(/improves confidence|less likely to reverse/i);
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Case 1 — Clear blocker control
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52F — Case 1: Clear blocker", () => {
+  it("maps to could_change_decision with match", () => {
+    const r = experimentResults["case1-clear-blocker"];
+    expect(r.returnedEnum).toBe(r.expectedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("grounds reasoning in the supplied statement only", () => {
+    const r = experimentResults["case1-clear-blocker"];
+    const reasonLower = r.reason.toLowerCase();
+    // The relationship says "cannot legally proceed" — the model should reference that, not invent external regulation facts
+    expect(reasonLower).toMatch(/legally.*proceed|cannot.*proceed|blocker|go\/no-go|must.*satisfy/);
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Case 2 — Clear support control
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52F — Case 2: Clear supporting evidence", () => {
+  it("maps to supports_decision with match", () => {
+    const r = experimentResults["case2-clear-support"];
+    expect(r.returnedEnum).toBe(r.expectedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("grounds reasoning in the supplied statement only", () => {
+    const r = experimentResults["case2-clear-support"];
+    const reasonLower = r.reason.toLowerCase();
+    // The relationship explicitly says "increase confidence" and "would not determine"
+    expect(reasonLower).toMatch(/confidence|evidence|improves.*support|not.*determine/);
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Case 3 — Genuinely ambiguous
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52F — Case 3: Ambiguous relationship", () => {
+  it("returns its enum result", () => {
+    const r = experimentResults["case3-ambiguous"];
+    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("expected enum is cannot_determine", () => {
+    const r = experimentResults["case3-ambiguous"];
+    expect(r.expectedEnum).toBe("cannot_determine");
+  });
+
+  it("grounding check for case 3", () => {
+    const r = experimentResults["case3-ambiguous"];
+    const grounding = checkGrounding(r.expectedEnum, r.reason, r.relationship);
+    expect(grounding).toBeTruthy();
+    // Log the result
+    console.log(`Case 3 grounding: ${grounding}`);
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Case 4 — Ambiguous condition
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52F — Case 4: Ambiguous condition", () => {
+  it("returns its enum result", () => {
+    const r = experimentResults["case4-ambiguous-condition"];
+    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("expected enum is cannot_determine", () => {
+    const r = experimentResults["case4-ambiguous-condition"];
+    expect(r.expectedEnum).toBe("cannot_determine");
+  });
+
+  it("grounding check for case 4", () => {
+    const r = experimentResults["case4-ambiguous-condition"];
+    const grounding = checkGrounding(r.expectedEnum, r.reason, r.relationship);
+    expect(grounding).toBeTruthy();
+    // Log the result
+    console.log(`Case 4 grounding: ${grounding}`);
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Overall results — match assessment
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52F — Overall results", () => {
+  it("reports all matches and mismatches", () => {
+    for (const c of CASES) {
+      const r = experimentResults[c.id];
+      const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
+      console.log(
+        `[${c.description}] ${c.id}: expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match} | ` +
+          `reason="${r.reason}" | latency=${r.latencyMs}ms`
+      );
+    }
+  });
+
+  it("clear controls (cases 1 & 2) both match", () => {
+    const r1 = experimentResults["case1-clear-blocker"];
+    const r2 = experimentResults["case2-clear-support"];
+    expect(r1.returnedEnum).toBe(r1.expectedEnum);
+    expect(r2.returnedEnum).toBe(r2.expectedEnum);
+  });
+
+  it("ambiguous cases (3 & 4) evaluate to cannot_determine", () => {
+    const r3 = experimentResults["case3-ambiguous"];
+    const r4 = experimentResults["case4-ambiguous-condition"];
+    // These assertions will fail if the model forces them into stronger categories
+    // — that failure is itself a finding we want to surface
+    expect(r3.returnedEnum).toBe("cannot_determine");
+    expect(r4.returnedEnum).toBe("cannot_determine");
+  });
+
+  it("clear-control match count", () => {
+    const r1 = experimentResults["case1-clear-blocker"];
+    const r2 = experimentResults["case2-clear-support"];
+    let clearMatchCount = 0;
+    if (r1.returnedEnum === r1.expectedEnum) clearMatchCount++;
+    if (r2.returnedEnum === r2.expectedEnum) clearMatchCount++;
+    expect(clearMatchCount).toBe(2);
+  });
+
+  it("cannot_determine count for ambiguous cases", () => {
+    const r3 = experimentResults["case3-ambiguous"];
+    const r4 = experimentResults["case4-ambiguous-condition"];
+    let canNotDetermineCount = 0;
+    if (r3.returnedEnum === "cannot_determine") canNotDetermineCount++;
+    if (r4.returnedEnum === "cannot_determine") canNotDetermineCount++;
+    expect(canNotDetermineCount).toBe(2);
+  });
+
+  it("external-assumption diagnostic for ambiguous cases", () => {
+    const r3 = experimentResults["case3-ambiguous"];
+    const r4 = experimentResults["case4-ambiguous-condition"];
+
+    const g3 = checkGrounding(r3.expectedEnum, r3.reason, r3.relationship);
+    const g4 = checkGrounding(r4.expectedEnum, r4.reason, r4.relationship);
+
+    console.log(`External assumption Case 3: ${g3}`);
+    console.log(`External assumption Case 4: ${g4}`);
+
+    // Document whether either case introduced external assumptions
+    const introducedAssumptions = [g3, g4].filter((g) => g === "introduced_external_assumption").length;
+    // We document but don't assert — the finding is informational
+    expect(typeof introducedAssumptions).toBe("number");
+  });
+
+  it("full output log", () => {
+    console.log("\n=== Experiment 52F Summary ===");
+    for (const c of CASES) {
+      const r = experimentResults[c.id];
+      const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
+      const grounding = checkGrounding(r.expectedEnum, r.reason, r.relationship);
+      console.log(
+        `[${c.description}] ${r.expectedEnum} → ${r.returnedEnum} (${match}) | ` +
+          `reason="${r.reason}" | grounding=${grounding} | latency=${r.latencyMs}ms`
+      );
+    }
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Inference timing (observational only)
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52F — Inference timing", () => {
+  it("records min, max, total timing for all 4 calls", () => {
+    expect(timingStats.min).toBeGreaterThan(0);
+    expect(timingStats.max).toBeGreaterThanOrEqual(timingStats.min);
+    expect(timingStats.total).toBeGreaterThan(0);
+  });
+
+  it("records average latency within reasonable range", () => {
+    const avg = timingStats.total / 4;
+    expect(avg).toBeGreaterThan(5000);
+    expect(avg).toBeLessThan(120000);
+  });
+
+  it("logs timing summary", () => {
+    const avg = Math.round(timingStats.total / 4);
+    console.log(`\n=== Experiment 52F Timing ===`);
+    console.log(`Calls: 4`);
+    console.log(`Total: ${timingStats.total}ms`);
+    console.log(`Average: ${avg}ms`);
+    console.log(`Fastest: ${timingStats.min}ms`);
+    console.log(`Slowest: ${timingStats.max}ms`);
+  });
+});