experiment: test wording effects on ambiguous relevance

2026-08-07 11:03:33 +01:00
parent 69d0216250
commit db5af97c98
3 changed files with 739 additions and 13 deletions
@@ -0,0 +1,576 @@
+/**
+ * Experiment 52H — Does Ambiguity Fail Because of the Word "Important," or Because the Model Resists `cannot_determine` More Generally?
+ *
+ * Passive ambiguity-language experiment. Holds domain and subject constant (market-entry / customer-demand)
+ * and varies only the ambiguous wording pattern. Tests whether the model converts different neutral phrases
+ * into decisive relevance categories, or whether it preserves `cannot_determine`.
+ *
+ * Five fixed relationship statements with identical underlying subject and decision.
+ * Uses exactly the same category definitions and normalisation instruction as Experiment 52G.
+ * Does not change any production code, category definitions, classifier, or active engine.
+ *
+ * Hypothesis: If `important` is the main cause, weaker phrasings may preserve `cannot_determine`.
+ * If the model generally dislikes leaving relevance unresolved, all phrasings will still strengthen.
+ */
+
+import dotenv from "dotenv";
+dotenv.config({ path: ".env.local" });
+
+import { describe, it, expect, beforeAll } from "vitest";
+
+/* ═══════════════════════════════════════════════════════════
+ * Enum categories (unchanged from production contract)
+ * ═══════════════════════════════════════════════════════════ */
+
+const ENUM_CATEGORIES = [
+  "could_change_decision",
+  "supports_decision",
+  "unlikely_to_change_decision",
+  "cannot_determine",
+];
+
+/* ═══════════════════════════════════════════════════════════
+ * Category definitions — identical to Experiment 52G and production
+ * ═══════════════════════════════════════════════════════════ */
+
+const CATEGORY_DEFINITIONS = {
+  could_change_decision:
+    "Answering could reasonably reverse the proposed action — it is a go/no-go condition or materially affects viability.",
+  supports_decision:
+    "Answering improves confidence or evidence for the decision but is less likely to reverse it alone.",
+  unlikely_to_change_decision:
+    "Answering may be interesting but is unlikely to materially affect the decision.",
+  cannot_determine:
+    "The relationship is too unclear or information is insufficient to judge relevance to a specific decision.",
+};
+
+/* ═══════════════════════════════════════════════════════════
+ * Normalisation instruction — identical to Experiment 52G (unchanged)
+ * ═══════════════════════════════════════════════════════════ */
+
+const NORMALISATION_INSTRUCTION = `You are given a short statement describing how an unanswered question relates to a decision. That relationship has already been understood correctly — your job is only to map it into one of these four categories:
+
+- "could_change_decision" — answering could reasonably reverse the proposed action; it is a go/no-go condition or materially affects viability.
+- "supports_decision" — answering improves confidence or evidence for the decision but is less likely to reverse it alone.
+- "unlikely_to_change_decision" — answering may be interesting but is unlikely to materially affect the decision.
+- "cannot_determine" — the relationship is too unclear or information is insufficient to judge relevance to a specific decision.
+
+Do not reinterpret the original situation — you have not been given it. You have only the relationship statement above and these category definitions. Choose the category that best matches the relationship statement.
+
+Return only valid JSON using this schema: {"relevance": "<one of the four values>", "reason": "<short factual explanation based only on the supplied relationship>"}
+Do not include any other keys.`;
+
+/* ═══════════════════════════════════════════════════════════
+ * Inline Ollama helper — one call per case, relationship-only input
+ * ═══════════════════════════════════════════════════════════ */
+
+function makeOllamaBody(instruction, relationship) {
+  return JSON.stringify({
+    model: process.env.OLLAMA_MODEL || "qwen-claude:latest",
+    messages: [
+      { role: "system", content: instruction },
+      {
+        role: "user",
+        content: `Relationship: "${relationship}"`,
+      },
+    ],
+    format: "json",
+    stream: false,
+  });
+}
+
+async function callAmbiguousWordingTest(relationship) {
+  const baseUrl = process.env.OLLAMA_BASE_URL;
+  if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");
+
+  const model = process.env.OLLAMA_MODEL || "qwen-claude:latest";
+  const body = makeOllamaBody(NORMALISATION_INSTRUCTION, relationship);
+  const res = await fetch(`${baseUrl}/api/chat`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body,
+    signal: AbortSignal.timeout(120000),
+  });
+  if (!res.ok) throw new Error(`Ollama returned ${res.status}`);
+  const data = await res.json();
+  const rawText =
+    typeof data.message?.content === "string"
+      ? data.message.content
+      : JSON.stringify(data.message?.content || {});
+  return { result: JSON.parse(rawText), model };
+}
+
+/* ═══════════════════════════════════════════════════════════
+ * Five fixed wording variants — same domain and subject, different wording
+ * All five communicate only "there is some relationship."
+ * None states how strong that relationship is.
+ * ═══════════════════════════════════════════════════════════ */
+
+const CASES = [
+  {
+    id: "case1-important",
+    wording: "\"important to\"",
+    description: "Important to (control — same pattern as Exp 52G)",
+    relationship: "Understanding customer demand would be important to the market-entry decision.",
+    expectedEnum: "cannot_determine",
+  },
+  {
+    id: "case2-relevant",
+    wording: "\"relevant to\"",
+    description: "Relevant to",
+    relationship: "Understanding customer demand would be relevant to the market-entry decision.",
+    expectedEnum: "cannot_determine",
+  },
+  {
+    id: "case3-worth",
+    wording: "\"worth considering\"",
+    description: "Worth considering",
+    relationship: "Customer demand would be worth considering when making the market-entry decision.",
+    expectedEnum: "cannot_determine",
+  },
+  {
+    id: "case4-maymatter",
+    wording: "\"may matter for\"",
+    description: "May matter for",
+    relationship: "Customer demand may matter for the market-entry decision.",
+    expectedEnum: "cannot_determine",
+  },
+  {
+    id: "case5-connected",
+    wording: "\"connected to\"",
+    description: "Connected to",
+    relationship: "Customer demand is connected to the market-entry decision.",
+    expectedEnum: "cannot_determine",
+  },
+];
+
+/* ═══════════════════════════════════════════════════════════
+ * Grounding diagnostic labels — test-only, not production
+ * ═══════════════════════════════════════════════════════════ */
+
+const GROUNDING_LABELS = {
+  GROUNDED_ONLY: "grounded_only_in_statement",
+  STRENGTHENED: "introduced_stronger_relationship",
+};
+
+function checkGrounding(expectedEnum, reason) {
+  if (expectedEnum !== "cannot_determine") return GROUNDING_LABELS.GROUNDED_ONLY;
+
+  const reasonLower = (reason || "").toLowerCase();
+  const strongerSignals = [
+    /material.*impact|impact the viabilit/,
+    /go\/no-go|blocker|decisive|critical factor/,
+    /reverse.*decision|reverses?.*action|change.*outcome/i,
+    /confidence.*strengthen|increase.*confident|matters for.*viable/,
+    /must.*satisfy|mandatory|essential.*condition|necessary.*for.*entry/,
+    /determines?.*viability|determine.*whether.*enter|determines?.*the decision/,
+    /unlikel?.*to.*proceed|prevent.*entry|block.*market/i,
+  ];
+
+  const hasStronger = strongerSignals.some((p) => p.test(reasonLower));
+  return hasStronger ? GROUNDING_LABELS.STRENGTHENED : GROUNDING_LABELS.GROUNDED_ONLY;
+}
+
+/* ═══════════════════════════════════════════════════════════
+ * Results holder — populated by beforeAll (5 calls total)
+ * ═══════════════════════════════════════════════════════════ */
+
+let experimentResults = {};
+let inferenceCount = 0;
+let timingStats = { min: Infinity, max: 0, total: 0 };
+let modelFailureReason = null;
+
+beforeAll(async () => {
+  experimentResults = {};
+
+  for (const c of CASES) {
+    let result = null;
+    let latency = 0;
+    const t0 = Date.now();
+    try {
+      result = await callAmbiguousWordingTest(c.relationship);
+      latency = Date.now() - t0;
+    } catch (e) {
+      modelFailureReason = `case ${c.id}: ${e.message}`;
+      result = { result: null };
+    }
+    timingStats.min = Math.min(timingStats.min, latency);
+    timingStats.max = Math.max(timingStats.max, latency);
+    timingStats.total += latency;
+
+    experimentResults[c.id] = {
+      wording: c.wording,
+      description: c.description,
+      relationship: c.relationship,
+      expectedEnum: c.expectedEnum,
+      returnedEnum: result.result?.relevance || "error",
+      reason: result.result?.reason || "none",
+      model: result.model,
+      latencyMs: latency,
+    };
+    inferenceCount += 1;
+  }
+}, 600000);
+
+/* ═══════════════════════════════════════════════════════════
+ * Infrastructure assertions — exactly 5 calls, same config, production unchanged
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52H — Infrastructure", () => {
+  it("uses Ollama config from .env.local", () => {
+    expect(process.env.OLLAMA_BASE_URL).toBeTruthy();
+    expect(process.env.OLLAMA_MODEL).toBe("qwen-claude:latest");
+  });
+
+  it("all returned enums belong to the existing four-category contract", () => {
+    for (const c of CASES) {
+      const r = experimentResults[c.id]?.returnedEnum;
+      expect(ENUM_CATEGORIES).toContain(r);
+    }
+  });
+
+  it("all cases include a reason string", () => {
+    for (const c of CASES) {
+      const r = experimentResults[c.id]?.reason;
+      expect(typeof r).toBe("string");
+      expect(r.length).toBeGreaterThan(0);
+    }
+  });
+
+  it("same Ollama host used throughout", () => {
+    expect(process.env.OLLAMA_BASE_URL).toBe("http://192.168.1.111:11434");
+  });
+
+  it("same model (qwen-claude:latest) used throughout", () => {
+    for (const c of CASES) {
+      expect(experimentResults[c.id]?.model).toBe("qwen-claude:latest");
+    }
+  });
+
+  it("exactly 5 live inference calls were made", () => {
+    expect(inferenceCount).toBe(5);
+  });
+
+  it("normalisation instruction is identical for all five calls", () => {
+    expect(typeof NORMALISATION_INSTRUCTION).toBe("string");
+    expect(NORMALISATION_INSTRUCTION.length).toBeGreaterThan(0);
+  });
+
+  it("fixed expected enums before any live call — all five are cannot_determine", () => {
+    let ambiguousCount = 0;
+    for (const c of CASES) {
+      if (c.expectedEnum === "cannot_determine") ambiguousCount++;
+    }
+    expect(ambiguousCount).toBe(5);
+  });
+
+  it("all five cases use the same domain and subject — only wording changes", () => {
+    // Verify all cases refer to customer demand / market-entry
+    for (const c of CASES) {
+      expect(c.relationship).toContain("market-entry");
+    }
+    // Verify wording differs between cases
+    const wordings = CASES.map((c) => c.wording);
+    const uniqueWordings = [...new Set(wordings)];
+    expect(uniqueWordings.length).toBe(5);
+  });
+
+  it("only the relationship statement is supplied to each case", () => {
+    for (const c of CASES) {
+      expect(c.relationship).toBeTruthy();
+      expect(typeof c.relationship).toBe("string");
+      expect(c).not.toHaveProperty("decisionTarget");
+      expect(c).not.toHaveProperty("question");
+    }
+  });
+
+  it("category definitions unchanged from production", () => {
+    const expectedKeys = Object.keys(CATEGORY_DEFINITIONS);
+    expect(expectedKeys).toContain("could_change_decision");
+    expect(expectedKeys).toContain("supports_decision");
+    expect(expectedKeys).toContain("unlikely_to_change_decision");
+    expect(expectedKeys).toContain("cannot_determine");
+
+    expect(CATEGORY_DEFINITIONS.could_change_decision).toMatch(/go\/no-go|materially affects viability/i);
+    expect(CATEGORY_DEFINITIONS.supports_decision).toMatch(/improves confidence|less likely to reverse/i);
+  });
+
+  it("instruction and definitions unchanged from Experiment 52G", () => {
+    const expectedInstruction = `You are given a short statement describing how an unanswered question relates to a decision. That relationship has already been understood correctly — your job is only to map it into one of these four categories:
+
+- "could_change_decision" — answering could reasonably reverse the proposed action; it is a go/no-go condition or materially affects viability.
+- "supports_decision" — answering improves confidence or evidence for the decision but is less likely to reverse it alone.
+- "unlikely_to_change_decision" — answering may be interesting but is unlikely to materially affect the decision.
+- "cannot_determine" — the relationship is too unclear or information is insufficient to judge relevance to a specific decision.
+
+Do not reinterpret the original situation — you have not been given it. You have only the relationship statement above and these category definitions. Choose the category that best matches the relationship statement.
+
+Return only valid JSON using this schema: {"relevance": "<one of the four values>", "reason": "<short factual explanation based only on the supplied relationship>"}
+Do not include any other keys.`;
+    expect(NORMALISATION_INSTRUCTION).toBe(expectedInstruction);
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Case 1 — "important to" (control)
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52H — Case 1: \"important to\"", () => {
+  it("returns its enum result within the contract", () => {
+    const r = experimentResults["case1-important"];
+    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("expected enum is cannot_determine", () => {
+    const r = experimentResults["case1-important"];
+    expect(r.expectedEnum).toBe("cannot_determine");
+  });
+
+  it("match/mismatch check", () => {
+    const r = experimentResults["case1-important"];
+    const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
+    console.log(`[important to] expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match}`);
+  });
+
+  it("grounding check", () => {
+    const r = experimentResults["case1-important"];
+    const grounding = checkGrounding(r.expectedEnum, r.reason);
+    console.log(`[important to] grounding: ${grounding} | reason="${r.reason}"`);
+    expect(grounding).toBeTruthy();
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Case 2 — "relevant to"
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52H — Case 2: \"relevant to\"", () => {
+  it("returns its enum result within the contract", () => {
+    const r = experimentResults["case2-relevant"];
+    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("expected enum is cannot_determine", () => {
+    const r = experimentResults["case2-relevant"];
+    expect(r.expectedEnum).toBe("cannot_determine");
+  });
+
+  it("match/mismatch check", () => {
+    const r = experimentResults["case2-relevant"];
+    const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
+    console.log(`[relevant to] expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match}`);
+  });
+
+  it("grounding check", () => {
+    const r = experimentResults["case2-relevant"];
+    const grounding = checkGrounding(r.expectedEnum, r.reason);
+    console.log(`[relevant to] grounding: ${grounding} | reason="${r.reason}"`);
+    expect(grounding).toBeTruthy();
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Case 3 — "worth considering"
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52H — Case 3: \"worth considering\"", () => {
+  it("returns its enum result within the contract", () => {
+    const r = experimentResults["case3-worth"];
+    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("expected enum is cannot_determine", () => {
+    const r = experimentResults["case3-worth"];
+    expect(r.expectedEnum).toBe("cannot_determine");
+  });
+
+  it("match/mismatch check", () => {
+    const r = experimentResults["case3-worth"];
+    const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
+    console.log(`[worth considering] expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match}`);
+  });
+
+  it("grounding check", () => {
+    const r = experimentResults["case3-worth"];
+    const grounding = checkGrounding(r.expectedEnum, r.reason);
+    console.log(`[worth considering] grounding: ${grounding} | reason="${r.reason}"`);
+    expect(grounding).toBeTruthy();
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Case 4 — "may matter for"
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52H — Case 4: \"may matter for\"", () => {
+  it("returns its enum result within the contract", () => {
+    const r = experimentResults["case4-maymatter"];
+    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("expected enum is cannot_determine", () => {
+    const r = experimentResults["case4-maymatter"];
+    expect(r.expectedEnum).toBe("cannot_determine");
+  });
+
+  it("match/mismatch check", () => {
+    const r = experimentResults["case4-maymatter"];
+    const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
+    console.log(`[may matter for] expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match}`);
+  });
+
+  it("grounding check", () => {
+    const r = experimentResults["case4-maymatter"];
+    const grounding = checkGrounding(r.expectedEnum, r.reason);
+    console.log(`[may matter for] grounding: ${grounding} | reason="${r.reason}"`);
+    expect(grounding).toBeTruthy();
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Case 5 — "connected to"
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52H — Case 5: \"connected to\"", () => {
+  it("returns its enum result within the contract", () => {
+    const r = experimentResults["case5-connected"];
+    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("expected enum is cannot_determine", () => {
+    const r = experimentResults["case5-connected"];
+    expect(r.expectedEnum).toBe("cannot_determine");
+  });
+
+  it("match/mismatch check", () => {
+    const r = experimentResults["case5-connected"];
+    const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
+    console.log(`[connected to] expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match}`);
+  });
+
+  it("grounding check", () => {
+    const r = experimentResults["case5-connected"];
+    const grounding = checkGrounding(r.expectedEnum, r.reason);
+    console.log(`[connected to] grounding: ${grounding} | reason="${r.reason}"`);
+    expect(grounding).toBeTruthy();
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Cross-wording comparison — does wording change classification?
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52H — Cross-wording analysis", () => {
+  it("reports all matches and mismatches", () => {
+    for (const c of CASES) {
+      const r = experimentResults[c.id];
+      const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
+      console.log(
+        `[${c.wording}] ${c.description}: expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match} | ` +
+          `reason="${r.reason}" | latency=${r.latencyMs}ms`
+      );
+    }
+  });
+
+  it("cannot_determine count for all five cases", () => {
+    let canNotDetermineCount = 0;
+    for (const c of CASES) {
+      const r = experimentResults[c.id];
+      if (r.returnedEnum === "cannot_determine") canNotDetermineCount++;
+    }
+    console.log(`Cannot_determine count: ${canNotDetermineCount}/5`);
+    expect(typeof canNotDetermineCount).toBe("number");
+  });
+
+  it("did \"important to\" (control) become could_change_decision?", () => {
+    const r = experimentResults["case1-important"];
+    const isCouldChange = r.returnedEnum === "could_change_decision";
+    console.log(`[important to] returned could_change_decision: ${isCouldChange}`);
+    expect(typeof isCouldChange).toBe("boolean");
+  });
+
+  it("did each weaker phrasing preserve cannot_determine?", () => {
+    const preserved = ["case2-relevant", "case3-worth", "case4-maymatter", "case5-connected"].map(
+      (id) => experimentResults[id].returnedEnum === "cannot_determine"
+    );
+    const allPreserved = preserved.every((v) => v);
+    console.log(`All weaker phrasings preserved cannot_determine: ${allPreserved}`);
+    console.log(`  relevant_to=${experimentResults["case2-relevant"].returnedEnum}`);
+    console.log(`  worth_considering=${experimentResults["case3-worth"].returnedEnum}`);
+    console.log(`  may_matter_for=${experimentResults["case4-maymatter"].returnedEnum}`);
+    console.log(`  connected_to=${experimentResults["case5-connected"].returnedEnum}`);
+    expect(typeof allPreserved).toBe("boolean");
+  });
+
+  it("did different wording produce different enum categories?", () => {
+    const categories = CASES.map((c) => experimentResults[c.id].returnedEnum);
+    const uniqueCategories = [...new Set(categories)];
+    const hadDivergence = uniqueCategories.length > 1;
+    console.log(`Unique categories across wordings: ${uniqueCategories.join(", ")} | divergence: ${hadDivergence}`);
+    expect(typeof hadDivergence).toBe("boolean");
+  });
+
+  it("count of cases where model introduced stronger relationship", () => {
+    let strengthenedCount = 0;
+    for (const c of CASES) {
+      const r = experimentResults[c.id];
+      if (r.expectedEnum === "cannot_determine") {
+        const grounding = checkGrounding(r.expectedEnum, r.reason);
+        if (grounding === GROUNDING_LABELS.STRENGTHENED) strengthenedCount++;
+      }
+    }
+    console.log(`Cases with stronger relationship introduced: ${strengthenedCount}/5`);
+    expect(typeof strengthenedCount).toBe("number");
+  });
+
+  it("full output log", () => {
+    console.log("\n=== Experiment 52H Summary ===");
+    for (const c of CASES) {
+      const r = experimentResults[c.id];
+      const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
+      const grounding = checkGrounding(r.expectedEnum, r.reason);
+      console.log(
+        `[${c.wording}] ${r.expectedEnum} → ${r.returnedEnum} (${match}) | ` +
+          `reason="${r.reason}" | grounding=${grounding} | latency=${r.latencyMs}ms`
+      );
+    }
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Inference timing (observational only)
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52H — Inference timing", () => {
+  it("records min, max, total timing for all 5 calls", () => {
+    expect(timingStats.min).toBeGreaterThan(0);
+    expect(timingStats.max).toBeGreaterThanOrEqual(timingStats.min);
+    expect(timingStats.total).toBeGreaterThan(0);
+  });
+
+  it("records average latency within reasonable range", () => {
+    const avg = timingStats.total / 5;
+    expect(avg).toBeGreaterThan(5000);
+    expect(avg).toBeLessThan(120000);
+  });
+
+  it("logs timing summary", () => {
+    const avg = Math.round(timingStats.total / 5);
+    console.log(`\n=== Experiment 52H Timing ===`);
+    console.log(`Calls: 5`);
+    console.log(`Total: ${timingStats.total}ms`);
+    console.log(`Average: ${avg}ms`);
+    console.log(`Fastest: ${timingStats.min}ms`);
+    console.log(`Slowest: ${timingStats.max}ms`);
+  });
+});