experiment: test domain priors in ambiguous relevance

2026-08-07 10:28:43 +01:00
parent 9171844f5b
commit 69d0216250
3 changed files with 719 additions and 3 deletions
@@ -0,0 +1,557 @@
+/**
+ * Experiment 52G — Does the Model Fill Ambiguous Meaning With Domain Expectations?
+ *
+ * Passive ambiguity/generalisation experiment. Tests whether ambiguous relationship statements
+ * are pushed toward different stronger categories depending on what the model believes matters
+ * in each domain, or whether they all remain `cannot_determine`.
+ *
+ * Four fixed ambiguous relationship statements with identical linguistic structure across four
+ * different domains. Uses exactly the same category definitions and normalisation instruction
+ * as Experiment 52F. Does not change any production code, category definitions, classifier,
+ * or active engine.
+ *
+ * Hypothesis: If the model generally fills semantic gaps from domain knowledge (not just
+ * regulatory priors), then different domains will produce different stronger categories despite
+ * having the same degree of explicitness in their statements.
+ */
+
+import dotenv from "dotenv";
+dotenv.config({ path: ".env.local" });
+
+import { describe, it, expect, beforeAll } from "vitest";
+
+/* ═══════════════════════════════════════════════════════════
+ * Enum categories (unchanged from production contract)
+ * ═══════════════════════════════════════════════════════════ */
+
+const ENUM_CATEGORIES = [
+  "could_change_decision",
+  "supports_decision",
+  "unlikely_to_change_decision",
+  "cannot_determine",
+];
+
+/* ═══════════════════════════════════════════════════════════
+ * Category definitions — identical to Experiment 52F and production
+ * ═══════════════════════════════════════════════════════════ */
+
+const CATEGORY_DEFINITIONS = {
+  could_change_decision:
+    "Answering could reasonably reverse the proposed action — it is a go/no-go condition or materially affects viability.",
+  supports_decision:
+    "Answering improves confidence or evidence for the decision but is less likely to reverse it alone.",
+  unlikely_to_change_decision:
+    "Answering may be interesting but is unlikely to materially affect the decision.",
+  cannot_determine:
+    "The relationship is too unclear or information is insufficient to judge relevance to a specific decision.",
+};
+
+/* ═══════════════════════════════════════════════════════════
+ * Normalisation instruction — identical to Experiment 52F (unchanged)
+ * ═══════════════════════════════════════════════════════════ */
+
+const NORMALISATION_INSTRUCTION = `You are given a short statement describing how an unanswered question relates to a decision. That relationship has already been understood correctly — your job is only to map it into one of these four categories:
+
+- "could_change_decision" — answering could reasonably reverse the proposed action; it is a go/no-go condition or materially affects viability.
+- "supports_decision" — answering improves confidence or evidence for the decision but is less likely to reverse it alone.
+- "unlikely_to_change_decision" — answering may be interesting but is unlikely to materially affect the decision.
+- "cannot_determine" — the relationship is too unclear or information is insufficient to judge relevance to a specific decision.
+
+Do not reinterpret the original situation — you have not been given it. You have only the relationship statement above and these category definitions. Choose the category that best matches the relationship statement.
+
+Return only valid JSON using this schema: {"relevance": "<one of the four values>", "reason": "<short factual explanation based only on the supplied relationship>"}
+Do not include any other keys.`;
+
+/* ═══════════════════════════════════════════════════════════
+ * Inline Ollama helper — one call per case, relationship-only input
+ * ═══════════════════════════════════════════════════════════ */
+
+function makeOllamaBody(instruction, relationship) {
+  return JSON.stringify({
+    model: process.env.OLLAMA_MODEL || "qwen-claude:latest",
+    messages: [
+      { role: "system", content: instruction },
+      {
+        role: "user",
+        content: `Relationship: "${relationship}"`,
+      },
+    ],
+    format: "json",
+    stream: false,
+  });
+}
+
+async function callDomainPriorTest(relationship) {
+  const baseUrl = process.env.OLLAMA_BASE_URL;
+  if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");
+
+  const model = process.env.OLLAMA_MODEL || "qwen-claude:latest";
+  const body = makeOllamaBody(NORMALISATION_INSTRUCTION, relationship);
+  const res = await fetch(`${baseUrl}/api/chat`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body,
+    signal: AbortSignal.timeout(120000),
+  });
+  if (!res.ok) throw new Error(`Ollama returned ${res.status}`);
+  const data = await res.json();
+  const rawText =
+    typeof data.message?.content === "string"
+      ? data.message.content
+      : JSON.stringify(data.message?.content || {});
+  return { result: JSON.parse(rawText), model };
+}
+
+/* ═══════════════════════════════════════════════════════════
+ * Four structurally matched ambiguous relationship statements
+ * Fixed before any model call. Expected enum is `cannot_determine` for all.
+ *
+ * Each uses the same linguistic template:
+ *   "Understanding [X] would be important to [decision]."
+ * ═══════════════════════════════════════════════════════════ */
+
+const CASES = [
+  {
+    id: "case1-regulation",
+    domain: "regulation",
+    description: "Regulation",
+    relationship:
+      "Understanding the regulatory position would be important to the market-entry decision.",
+    expectedEnum: "cannot_determine",
+  },
+  {
+    id: "case2-weather",
+    domain: "weather",
+    description: "Weather",
+    relationship:
+      "Understanding the weather outlook would be important to the outdoor-event decision.",
+    expectedEnum: "cannot_determine",
+  },
+  {
+    id: "case3-employment-references",
+    domain: "employment",
+    description: "Employment References",
+    relationship:
+      "Understanding what the candidate's references say would be important to the hiring decision.",
+    expectedEnum: "cannot_determine",
+  },
+  {
+    id: "case4-customer-feedback",
+    domain: "customer-feedback",
+    description: "Customer Feedback",
+    relationship:
+      "Understanding what customers think would be important to the product-launch decision.",
+    expectedEnum: "cannot_determine",
+  },
+];
+
+/* ═══════════════════════════════════════════════════════════
+ * External-assumption diagnostic labels
+ * ═══════════════════════════════════════════════════════════ */
+
+const GROUNDING_LABELS = {
+  GROUNDED_ONLY: "grounded_only_in_statement",
+  EXTERNAL_ASSUMPTION: "introduced_external_assumption",
+};
+
+/* ═══════════════════════════════════════════════════════════
+ * Results holder — populated by beforeAll (4 calls total)
+ * ═══════════════════════════════════════════════════════════ */
+
+let experimentResults = {};
+let inferenceCount = 0;
+let timingStats = { min: Infinity, max: 0, total: 0 };
+let modelFailureReason = null;
+
+beforeAll(async () => {
+  experimentResults = {};
+
+  for (const c of CASES) {
+    let result = null;
+    let latency = 0;
+    const t0 = Date.now();
+    try {
+      result = await callDomainPriorTest(c.relationship);
+      latency = Date.now() - t0;
+    } catch (e) {
+      modelFailureReason = `case ${c.id}: ${e.message}`;
+      result = { result: null };
+    }
+    timingStats.min = Math.min(timingStats.min, latency);
+    timingStats.max = Math.max(timingStats.max, latency);
+    timingStats.total += latency;
+
+    experimentResults[c.id] = {
+      domain: c.domain,
+      description: c.description,
+      relationship: c.relationship,
+      expectedEnum: c.expectedEnum,
+      returnedEnum: result.result?.relevance || "error",
+      reason: result.result?.reason || "none",
+      model: result.model,
+      latencyMs: latency,
+    };
+    inferenceCount += 1;
+  }
+}, 600000);
+
+/* ═══════════════════════════════════════════════════════════
+ * Infrastructure assertions — exactly 4 calls, same config, production unchanged
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52G — Infrastructure", () => {
+  it("uses Ollama config from .env.local", () => {
+    expect(process.env.OLLAMA_BASE_URL).toBeTruthy();
+    expect(process.env.OLLAMA_MODEL).toBe("qwen-claude:latest");
+  });
+
+  it("all returned enums belong to the existing four-category contract", () => {
+    for (const c of CASES) {
+      const r = experimentResults[c.id]?.returnedEnum;
+      expect(ENUM_CATEGORIES).toContain(r);
+    }
+  });
+
+  it("all cases include a reason string", () => {
+    for (const c of CASES) {
+      const r = experimentResults[c.id]?.reason;
+      expect(typeof r).toBe("string");
+      expect(r.length).toBeGreaterThan(0);
+    }
+  });
+
+  it("same Ollama host used throughout", () => {
+    expect(process.env.OLLAMA_BASE_URL).toBe("http://192.168.1.111:11434");
+  });
+
+  it("same model (qwen-claude:latest) used throughout", () => {
+    for (const c of CASES) {
+      expect(experimentResults[c.id]?.model).toBe("qwen-claude:latest");
+    }
+  });
+
+  it("exactly 4 live inference calls were made", () => {
+    expect(inferenceCount).toBe(4);
+  });
+
+  it("normalisation instruction is identical for all four calls", () => {
+    expect(typeof NORMALISATION_INSTRUCTION).toBe("string");
+    expect(NORMALISATION_INSTRUCTION.length).toBeGreaterThan(0);
+  });
+
+  it("fixed expected enums before any live call — all four are cannot_determine", () => {
+    let ambiguousCount = 0;
+    for (const c of CASES) {
+      if (c.expectedEnum === "cannot_determine") ambiguousCount++;
+    }
+    expect(ambiguousCount).toBe(4);
+  });
+
+  it("all four cases use closely matched linguistic structure", () => {
+    const patterns = CASES.map((c) => c.relationship);
+    // All should contain "Understanding" and "would be important to"
+    for (const p of patterns) {
+      expect(p).toMatch(/Understanding.*would be important to/i);
+      expect(p).toMatch(/decision\.$/i);
+    }
+  });
+
+  it("only the relationship statement is supplied to each case", () => {
+    for (const c of CASES) {
+      expect(c.relationship).toBeTruthy();
+      expect(typeof c.relationship).toBe("string");
+      expect(c).not.toHaveProperty("decisionTarget");
+      expect(c).not.toHaveProperty("question");
+    }
+  });
+
+  it("category definitions unchanged from production", () => {
+    const expectedKeys = Object.keys(CATEGORY_DEFINITIONS);
+    expect(expectedKeys).toContain("could_change_decision");
+    expect(expectedKeys).toContain("supports_decision");
+    expect(expectedKeys).toContain("unlikely_to_change_decision");
+    expect(expectedKeys).toContain("cannot_determine");
+
+    expect(CATEGORY_DEFINITIONS.could_change_decision).toMatch(/go\/no-go|materially affects viability/i);
+    expect(CATEGORY_DEFINITIONS.supports_decision).toMatch(/improves confidence|less likely to reverse/i);
+  });
+
+  it("instruction and definitions unchanged from Experiment 52F", () => {
+    const expectedInstruction = `You are given a short statement describing how an unanswered question relates to a decision. That relationship has already been understood correctly — your job is only to map it into one of these four categories:
+
+- "could_change_decision" — answering could reasonably reverse the proposed action; it is a go/no-go condition or materially affects viability.
+- "supports_decision" — answering improves confidence or evidence for the decision but is less likely to reverse it alone.
+- "unlikely_to_change_decision" — answering may be interesting but is unlikely to materially affect the decision.
+- "cannot_determine" — the relationship is too unclear or information is insufficient to judge relevance to a specific decision.
+
+Do not reinterpret the original situation — you have not been given it. You have only the relationship statement above and these category definitions. Choose the category that best matches the relationship statement.
+
+Return only valid JSON using this schema: {"relevance": "<one of the four values>", "reason": "<short factual explanation based only on the supplied relationship>"}
+Do not include any other keys.`;
+    expect(NORMALISATION_INSTRUCTION).toBe(expectedInstruction);
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Case 1 — Regulation
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52G — Case 1: Regulation", () => {
+  it("returns its enum result within the contract", () => {
+    const r = experimentResults["case1-regulation"];
+    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("expected enum is cannot_determine", () => {
+    const r = experimentResults["case1-regulation"];
+    expect(r.expectedEnum).toBe("cannot_determine");
+  });
+
+  it("match/mismatch check", () => {
+    const r = experimentResults["case1-regulation"];
+    const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
+    console.log(`[Regulation] expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match}`);
+  });
+
+  it("external-assumption diagnostic", () => {
+    const r = experimentResults["case1-regulation"];
+    // Check for regulation-specific external assumptions (treats "important" as go/no-go blocker)
+    const reasonLower = r.reason.toLowerCase();
+    const hasBlockerAssumption = /regulatory.*always|compliance.*mandatory|regulation.*must.*block|legal.*prerequisite|cannot proceed without|legally mandatory|by definition.*blocking/i.test(reasonLower);
+    const grounding = hasBlockerAssumption ? "introduced_external_assumption" : "grounded_only_in_statement";
+    console.log(`[Regulation] external-assumption: ${grounding}`);
+    expect(grounding).toBeTruthy();
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Case 2 — Weather
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52G — Case 2: Weather", () => {
+  it("returns its enum result within the contract", () => {
+    const r = experimentResults["case2-weather"];
+    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("expected enum is cannot_determine", () => {
+    const r = experimentResults["case2-weather"];
+    expect(r.expectedEnum).toBe("cannot_determine");
+  });
+
+  it("match/mismatch check", () => {
+    const r = experimentResults["case2-weather"];
+    const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
+    console.log(`[Weather] expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match}`);
+  });
+
+  it("external-assumption diagnostic", () => {
+    const r = experimentResults["case2-weather"];
+    // Check for weather-specific external assumptions (treats "important" as cancellation risk)
+    const reasonLower = r.reason.toLowerCase();
+    const hasBlockerAssumption = /weather.*automatically|bad weather.*impossible|dangerous.*cancels|extreme.*force.*cancel|necessarily.*prevent/i.test(reasonLower);
+    const grounding = hasBlockerAssumption ? "introduced_external_assumption" : "grounded_only_in_statement";
+    console.log(`[Weather] external-assumption: ${grounding}`);
+    expect(grounding).toBeTruthy();
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Case 3 — Employment References
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52G — Case 3: Employment References", () => {
+  it("returns its enum result within the contract", () => {
+    const r = experimentResults["case3-employment-references"];
+    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("expected enum is cannot_determine", () => {
+    const r = experimentResults["case3-employment-references"];
+    expect(r.expectedEnum).toBe("cannot_determine");
+  });
+
+  it("match/mismatch check", () => {
+    const r = experimentResults["case3-employment-references"];
+    const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
+    console.log(`[Employment References] expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match}`);
+  });
+
+  it("external-assumption diagnostic", () => {
+    const r = experimentResults["case3-employment-references"];
+    // Check for employment-specific external assumptions (treats "important" as decisive)
+    const reasonLower = r.reason.toLowerCase();
+    const hasBlockerAssumption = /references.*determine|hiring decision depends|mandatory.*reference|must.*pass reference|reference.*disqualify|automatically.*suitable/i.test(reasonLower);
+    const grounding = hasBlockerAssumption ? "introduced_external_assumption" : "grounded_only_in_statement";
+    console.log(`[Employment References] external-assumption: ${grounding}`);
+    expect(grounding).toBeTruthy();
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Case 4 — Customer Feedback
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52G — Case 4: Customer Feedback", () => {
+  it("returns its enum result within the contract", () => {
+    const r = experimentResults["case4-customer-feedback"];
+    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("expected enum is cannot_determine", () => {
+    const r = experimentResults["case4-customer-feedback"];
+    expect(r.expectedEnum).toBe("cannot_determine");
+  });
+
+  it("match/mismatch check", () => {
+    const r = experimentResults["case4-customer-feedback"];
+    const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
+    console.log(`[Customer Feedback] expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match}`);
+  });
+
+  it("external-assumption diagnostic", () => {
+    const r = experimentResults["case4-customer-feedback"];
+    // Check for customer-feedback-specific external assumptions (treats "important" as viability blocker)
+    const reasonLower = r.reason.toLowerCase();
+    const hasBlockerAssumption = /customer.*stop.*launch|feedback.*determines.*viability|must.*cancel.*product|necessarily.*block.*launch/i.test(reasonLower);
+    const grounding = hasBlockerAssumption ? "introduced_external_assumption" : "grounded_only_in_statement";
+    console.log(`[Customer Feedback] external-assumption: ${grounding}`);
+    expect(grounding).toBeTruthy();
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Cross-domain comparison — does domain affect category choice?
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52G — Cross-domain analysis", () => {
+  it("reports all matches and mismatches", () => {
+    for (const c of CASES) {
+      const r = experimentResults[c.id];
+      const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
+      console.log(
+        `[${c.description}] ${c.domain}: expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match} | ` +
+          `reason="${r.reason}" | latency=${r.latencyMs}ms`
+      );
+    }
+  });
+
+  it("cannot_determine count for all four cases", () => {
+    let canNotDetermineCount = 0;
+    for (const c of CASES) {
+      const r = experimentResults[c.id];
+      if (r.returnedEnum === "cannot_determine") canNotDetermineCount++;
+    }
+    console.log(`Cannot_determine count: ${canNotDetermineCount}/4`);
+    expect(typeof canNotDetermineCount).toBe("number");
+  });
+
+  it("does regulation return could_change_decision again?", () => {
+    const r = experimentResults["case1-regulation"];
+    const isCouldChange = r.returnedEnum === "could_change_decision";
+    console.log(`[Regulation] returned could_change_decision: ${isCouldChange}`);
+    expect(typeof isCouldChange).toBe("boolean");
+  });
+
+  it("did different domains produce different stronger categories?", () => {
+    const categories = CASES.map((c) => experimentResults[c.id].returnedEnum);
+    const uniqueCategories = [...new Set(categories)];
+    const hadDivergence = uniqueCategories.length > 1;
+    console.log(`Unique categories across domains: ${uniqueCategories.join(", ")} | divergence: ${hadDivergence}`);
+    expect(typeof hadDivergence).toBe("boolean");
+  });
+
+  it("count of cases where model introduced external assumptions", () => {
+    let assumedCount = 0;
+    for (const c of CASES) {
+      const r = experimentResults[c.id];
+      if (r.expectedEnum === "cannot_determine") {
+        const reasonLower = r.reason.toLowerCase();
+        const hasAssumption = (() => {
+          switch (c.domain) {
+            case "regulation":
+              return /regulatory.*always|compliance.*mandatory|regulation.*must.*block|legal.*prerequisite|cannot proceed without|legally mandatory|by definition.*blocking/i.test(reasonLower);
+            case "weather":
+              return /weather.*automatically|bad weather.*impossible|dangerous.*cancels|extreme.*force.*cancel|necessarily.*prevent/i.test(reasonLower);
+            case "employment":
+              return /references.*determine|hiring decision depends|mandatory.*reference|must.*pass reference|reference.*disqualify|automatically.*suitable/i.test(reasonLower);
+            case "customer-feedback":
+              return /customer.*stop.*launch|feedback.*determines.*viability|must.*cancel.*product|necessarily.*block.*launch/i.test(reasonLower);
+            default:
+              return false;
+          }
+        })();
+        if (hasAssumption) assumedCount++;
+      }
+    }
+    console.log(`Cases with external assumptions: ${assumedCount}/4`);
+    expect(typeof assumedCount).toBe("number");
+  });
+
+  it("full output log", () => {
+    console.log("\n=== Experiment 52G Summary ===");
+    for (const c of CASES) {
+      const r = experimentResults[c.id];
+      const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
+      const reasonLower = r.reason.toLowerCase();
+      let grounding = "grounded_only_in_statement";
+      if (r.expectedEnum === "cannot_determine") {
+        const hasAssumption = (() => {
+          switch (c.domain) {
+            case "regulation":
+              return /regulatory.*always|compliance.*mandatory|regulation.*must.*block|legal.*prerequisite|cannot proceed without|legally mandatory|by definition.*blocking/i.test(reasonLower);
+            case "weather":
+              return /weather.*automatically|bad weather.*impossible|dangerous.*cancels|extreme.*force.*cancel|necessarily.*prevent/i.test(reasonLower);
+            case "employment":
+              return /references.*determine|hiring decision depends|mandatory.*reference|must.*pass reference|reference.*disqualify|automatically.*suitable/i.test(reasonLower);
+            case "customer-feedback":
+              return /customer.*stop.*launch|feedback.*determines.*viability|must.*cancel.*product|necessarily.*block.*launch/i.test(reasonLower);
+            default:
+              return false;
+          }
+        })();
+        if (hasAssumption) grounding = "introduced_external_assumption";
+      }
+      console.log(
+        `[${c.description}] ${r.expectedEnum} → ${r.returnedEnum} (${match}) | ` +
+          `reason="${r.reason}" | domain_assumption=${grounding} | latency=${r.latencyMs}ms`
+      );
+    }
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Inference timing (observational only)
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52G — Inference timing", () => {
+  it("records min, max, total timing for all 4 calls", () => {
+    expect(timingStats.min).toBeGreaterThan(0);
+    expect(timingStats.max).toBeGreaterThanOrEqual(timingStats.min);
+    expect(timingStats.total).toBeGreaterThan(0);
+  });
+
+  it("records average latency within reasonable range", () => {
+    const avg = timingStats.total / 4;
+    expect(avg).toBeGreaterThan(5000);
+    expect(avg).toBeLessThan(120000);
+  });
+
+  it("logs timing summary", () => {
+    const avg = Math.round(timingStats.total / 4);
+    console.log(`\n=== Experiment 52G Timing ===`);
+    console.log(`Calls: 4`);
+    console.log(`Total: ${timingStats.total}ms`);
+    console.log(`Average: ${avg}ms`);
+    console.log(`Fastest: ${timingStats.min}ms`);
+    console.log(`Slowest: ${timingStats.max}ms`);
+  });
+});