experiment: test decision-relevance category boundary

2026-08-07 10:02:03 +01:00
parent 34f06f2919
commit 08c8f74bde
3 changed files with 572 additions and 3 deletions
@@ -0,0 +1,421 @@
+/**
+ * Experiment 52E — Is the `supports_decision` / `could_change_decision` Boundary Actually Coherent?
+ *
+ * Passive contract-boundary experiment. Tests whether the existing distinction between
+ * `could_change_decision` and `supports_decision` holds consistently across three explicit
+ * contrast pairs (blocker vs supporting-evidence) in three distinct domains.
+ *
+ * Uses exactly the same category definitions and normalisation instruction as Experiment 52D.
+ * Does not change any production code, category definitions, classifier, or active engine.
+ */
+
+import dotenv from "dotenv";
+dotenv.config({ path: ".env.local" });
+
+import { describe, it, expect, beforeAll } from "vitest";
+
+/* ═══════════════════════════════════════════════════════════
+ * Enum categories (unchanged from production contract)
+ * ═══════════════════════════════════════════════════════════ */
+
+const ENUM_CATEGORIES = [
+  "could_change_decision",
+  "supports_decision",
+  "unlikely_to_change_decision",
+  "cannot_determine",
+];
+
+/* ═══════════════════════════════════════════════════════════
+ * Category definitions — identical to Experiment 52D and production
+ * ═══════════════════════════════════════════════════════════ */
+
+const CATEGORY_DEFINITIONS = {
+  could_change_decision:
+    "Answering could reasonably reverse the proposed action — it is a go/no-go condition or materially affects viability.",
+  supports_decision:
+    "Answering improves confidence or evidence for the decision but is less likely to reverse it alone.",
+  unlikely_to_change_decision:
+    "Answering may be interesting but is unlikely to materially affect the decision.",
+  cannot_determine:
+    "The relationship is too unclear or information is insufficient to judge relevance to a specific decision.",
+};
+
+/* ═══════════════════════════════════════════════════════════
+ * Normalisation instruction — identical to Experiment 52D
+ * ═══════════════════════════════════════════════════════════ */
+
+const NORMALISATION_INSTRUCTION = `You are given a short statement describing how an unanswered question relates to a decision. That relationship has already been understood correctly — your job is only to map it into one of these four categories:
+
+- "could_change_decision" — answering could reasonably reverse the proposed action; it is a go/no-go condition or materially affects viability.
+- "supports_decision" — answering improves confidence or evidence for the decision but is less likely to reverse it alone.
+- "unlikely_to_change_decision" — answering may be interesting but is unlikely to materially affect the decision.
+- "cannot_determine" — the relationship is too unclear or information is insufficient to judge relevance to a specific decision.
+
+Do not reinterpret the original situation — you have not been given it. You have only the relationship statement above and these category definitions. Choose the category that best matches the relationship statement.
+
+Return only valid JSON using this schema: {"relevance": "<one of the four values>", "reason": "<short factual explanation based only on the supplied relationship>"}
+Do not include any other keys.`;
+
+/* ═══════════════════════════════════════════════════════════
+ * Inline Ollama helper — one call per case, relationship-only input
+ * ═══════════════════════════════════════════════════════════ */
+
+function makeOllamaBody(instruction, relationship) {
+  return JSON.stringify({
+    model: process.env.OLLAMA_MODEL || "qwen-claude:latest",
+    messages: [
+      { role: "system", content: instruction },
+      {
+        role: "user",
+        content: `Relationship: "${relationship}"`,
+      },
+    ],
+    format: "json",
+    stream: false,
+  });
+}
+
+async function callBoundaryTest(relationship) {
+  const baseUrl = process.env.OLLAMA_BASE_URL;
+  if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");
+
+  const model = process.env.OLLAMA_MODEL || "qwen-claude:latest";
+  const body = makeOllamaBody(NORMALISATION_INSTRUCTION, relationship);
+  const res = await fetch(`${baseUrl}/api/chat`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body,
+    signal: AbortSignal.timeout(120000),
+  });
+  if (!res.ok) throw new Error(`Ollama returned ${res.status}`);
+  const data = await res.json();
+  const rawText =
+    typeof data.message?.content === "string"
+      ? data.message.content
+      : JSON.stringify(data.message?.content || {});
+  return { result: JSON.parse(rawText), model };
+}
+
+/* ═══════════════════════════════════════════════════════════
+ * Three contrast pairs — six fixed relationship statements
+ * Fixed before any model call. Expected enums decided before testing.
+ * ═══════════════════════════════════════════════════════════ */
+
+const CONTRAST_PAIRS = [
+  {
+    pair: "Pair 1 (Market Entry)",
+    blocker: {
+      id: "1A-blocker",
+      relationship:
+        "If the product cannot legally satisfy the required European regulations, entering the market cannot proceed.",
+      expectedEnum: "could_change_decision",
+    },
+    supporting: {
+      id: "1B-supporting",
+      relationship:
+        "Independent customer interviews showing strong interest would increase confidence that entering the European market is worthwhile, but would not determine the decision by themselves.",
+      expectedEnum: "supports_decision",
+    },
+  },
+  {
+    pair: "Pair 2 (Community Event)",
+    blocker: {
+      id: "2A-blocker",
+      relationship:
+        "If the forecast shows dangerous weather conditions on the event date, holding the event outdoors would no longer be viable.",
+      expectedEnum: "could_change_decision",
+    },
+    supporting: {
+      id: "2B-supporting",
+      relationship:
+        "Positive feedback from previous attendees about outdoor events would strengthen confidence in choosing an outdoor venue, but would not decide the issue by itself.",
+      expectedEnum: "supports_decision",
+    },
+  },
+  {
+    pair: "Pair 3 (Hiring Decision)",
+    blocker: {
+      id: "3A-blocker",
+      relationship:
+        "If the candidate does not hold the legally required professional licence, they cannot be appointed to the role.",
+      expectedEnum: "could_change_decision",
+    },
+    supporting: {
+      id: "3B-supporting",
+      relationship:
+        "Strong references from previous employers would increase confidence that the candidate is suitable, but would not determine the hiring decision alone.",
+      expectedEnum: "supports_decision",
+    },
+  },
+];
+
+/* Flatten all six cases */
+const ALL_CASES = [];
+for (const pair of CONTRAST_PAIRS) {
+  ALL_CASES.push({ ...pair.blocker, pairLabel: pair.pair });
+  ALL_CASES.push({ ...pair.supporting, pairLabel: pair.pair });
+}
+
+/* ═══════════════════════════════════════════════════════════
+ * Results holder — populated by beforeAll (6 calls total)
+ * ═══════════════════════════════════════════════════════════ */
+
+let experimentResults = {};
+let inferenceCount = 0;
+let timingStats = { min: Infinity, max: 0, total: 0 };
+let modelFailureReason = null;
+
+beforeAll(async () => {
+  experimentResults = {};
+
+  for (const c of ALL_CASES) {
+    let result = null;
+    let latency = 0;
+    const t0 = Date.now();
+    try {
+      result = await callBoundaryTest(c.relationship);
+      latency = Date.now() - t0;
+    } catch (e) {
+      modelFailureReason = `case ${c.id}: ${e.message}`;
+      result = { result: null };
+    }
+    timingStats.min = Math.min(timingStats.min, latency);
+    timingStats.max = Math.max(timingStats.max, latency);
+    timingStats.total += latency;
+
+    experimentResults[c.id] = {
+      pair: c.pairLabel,
+      relationship: c.relationship,
+      expectedEnum: c.expectedEnum,
+      returnedEnum: result.result?.relevance || "error",
+      reason: result.result?.reason || "none",
+      model: result.model,
+      latencyMs: latency,
+    };
+    inferenceCount += 1;
+  }
+}, 600000);
+
+/* ═══════════════════════════════════════════════════════════
+ * Infrastructure assertions — exactly 6 calls, same config, production unchanged
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52E — Infrastructure", () => {
+  it("uses Ollama config from .env.local", () => {
+    expect(process.env.OLLAMA_BASE_URL).toBeTruthy();
+    expect(process.env.OLLAMA_MODEL).toBe("qwen-claude:latest");
+  });
+
+  it("all returned enums belong to the existing four-category contract", () => {
+    for (const c of ALL_CASES) {
+      const r = experimentResults[c.id]?.returnedEnum;
+      expect(ENUM_CATEGORIES).toContain(r);
+    }
+  });
+
+  it("all cases include a reason string", () => {
+    for (const c of ALL_CASES) {
+      const r = experimentResults[c.id]?.reason;
+      expect(typeof r).toBe("string");
+      expect(r.length).toBeGreaterThan(0);
+    }
+  });
+
+  it("same Ollama host used throughout", () => {
+    expect(process.env.OLLAMA_BASE_URL).toBe("http://192.168.1.111:11434");
+  });
+
+  it("same model (qwen-claude:latest) used throughout", () => {
+    for (const c of ALL_CASES) {
+      expect(experimentResults[c.id]?.model).toBe("qwen-claude:latest");
+    }
+  });
+
+  it("exactly 6 live inference calls were made", () => {
+    expect(inferenceCount).toBe(6);
+  });
+
+  it("normalisation instruction is identical for all six calls", () => {
+    expect(typeof NORMALISATION_INSTRUCTION).toBe("string");
+    expect(NORMALISATION_INSTRUCTION.length).toBeGreaterThan(0);
+  });
+
+  it("fixed expected enums before any live call — structure check", () => {
+    // Verify the fixed expected values match our design: 3 blockers = could_change_decision, 3 supporting = supports_decision
+    let blockerCount = 0;
+    let supportingCount = 0;
+    for (const c of ALL_CASES) {
+      if (c.expectedEnum === "could_change_decision") blockerCount++;
+      if (c.expectedEnum === "supports_decision") supportingCount++;
+    }
+    expect(blockerCount).toBe(3);
+    expect(supportingCount).toBe(3);
+  });
+
+  it("only the relationship statement is supplied to each case", () => {
+    for (const c of ALL_CASES) {
+      expect(c.relationship).toBeTruthy();
+      expect(typeof c.relationship).toBe("string");
+      // No decision target or question keys in the test data
+      expect(c).not.toHaveProperty("decisionTarget");
+      expect(c).not.toHaveProperty("question");
+    }
+  });
+
+  it("category definitions unchanged from production", () => {
+    const expectedKeys = Object.keys(CATEGORY_DEFINITIONS);
+    expect(expectedKeys).toContain("could_change_decision");
+    expect(expectedKeys).toContain("supports_decision");
+    expect(expectedKeys).toContain("unlikely_to_change_decision");
+    expect(expectedKeys).toContain("cannot_determine");
+
+    // Verify definitions match production wording
+    expect(CATEGORY_DEFINITIONS.could_change_decision).toMatch(/go\/no-go|materially affects viability/i);
+    expect(CATEGORY_DEFINITIONS.supports_decision).toMatch(/improves confidence|less likely to reverse/i);
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Blocker results — all three should map to could_change_decision
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52E — Blocker classification (could_change_decision)", () => {
+  it("Pair 1A blocker maps to could_change_decision", () => {
+    const r = experimentResults["1A-blocker"];
+    expect(r.returnedEnum).toBe(r.expectedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("Pair 2A blocker maps to could_change_decision", () => {
+    const r = experimentResults["2A-blocker"];
+    expect(r.returnedEnum).toBe(r.expectedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("Pair 3A blocker maps to could_change_decision", () => {
+    const r = experimentResults["3A-blocker"];
+    expect(r.returnedEnum).toBe(r.expectedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("all three blockers consistently map to could_change_decision", () => {
+    const blockerResults = ALL_CASES.filter((c) => c.expectedEnum === "could_change_decision");
+    for (const c of blockerResults) {
+      expect(experimentResults[c.id].returnedEnum).toBe("could_change_decision");
+    }
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Supporting-evidence results — all three should map to supports_decision
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52E — Supporting evidence classification (supports_decision)", () => {
+  it("Pair 1B supporting maps to supports_decision", () => {
+    const r = experimentResults["1B-supporting"];
+    expect(r.returnedEnum).toBe(r.expectedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("Pair 2B supporting maps to supports_decision", () => {
+    const r = experimentResults["2B-supporting"];
+    expect(r.returnedEnum).toBe(r.expectedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("Pair 3B supporting maps to supports_decision", () => {
+    const r = experimentResults["3B-supporting"];
+    expect(r.returnedEnum).toBe(r.expectedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("all three supporting-evidence cases consistently map to supports_decision", () => {
+    const supportingResults = ALL_CASES.filter((c) => c.expectedEnum === "supports_decision");
+    for (const c of supportingResults) {
+      expect(experimentResults[c.id].returnedEnum).toBe("supports_decision");
+    }
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Contrast-pair consistency — each pair's blocker vs supporting must differ
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52E — Contrast-pair consistency", () => {
+  it("Pair 1: blocker and supporting map to different categories", () => {
+    const rA = experimentResults["1A-blocker"].returnedEnum;
+    const rB = experimentResults["1B-supporting"].returnedEnum;
+    expect(rA).not.toBe(rB);
+  });
+
+  it("Pair 2: blocker and supporting map to different categories", () => {
+    const rA = experimentResults["2A-blocker"].returnedEnum;
+    const rB = experimentResults["2B-supporting"].returnedEnum;
+    expect(rA).not.toBe(rB);
+  });
+
+  it("Pair 3: blocker and supporting map to different categories", () => {
+    const rA = experimentResults["3A-blocker"].returnedEnum;
+    const rB = experimentResults["3B-supporting"].returnedEnum;
+    expect(rA).not.toBe(rB);
+  });
+
+  it("no supporting evidence was classified as could_change_decision (decision-reverser misclassification)", () => {
+    const supportingCases = ALL_CASES.filter((c) => c.expectedEnum === "supports_decision");
+    for (const c of supportingCases) {
+      expect(experimentResults[c.id].returnedEnum).not.toBe("could_change_decision");
+    }
+  });
+
+  it("no blocker was classified as supports_decision (blocker downgraded to supportive)", () => {
+    const blockerCases = ALL_CASES.filter((c) => c.expectedEnum === "could_change_decision");
+    for (const c of blockerCases) {
+      expect(experimentResults[c.id].returnedEnum).not.toBe("supports_decision");
+    }
+  });
+
+  it("full output log", () => {
+    for (const c of ALL_CASES) {
+      const r = experimentResults[c.id];
+      const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
+      console.log(
+        `[${r.pair}] ${c.id}: expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match} | ` +
+          `reason="${r.reason}" | latency=${r.latencyMs}ms`
+      );
+    }
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Inference timing (observational only)
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52E — Inference timing", () => {
+  it("records min, max, total timing for all 6 calls", () => {
+    expect(timingStats.min).toBeGreaterThan(0);
+    expect(timingStats.max).toBeGreaterThanOrEqual(timingStats.min);
+    expect(timingStats.total).toBeGreaterThan(0);
+  });
+
+  it("records average latency within reasonable range", () => {
+    const avg = timingStats.total / 6;
+    expect(avg).toBeGreaterThan(5000);
+    expect(avg).toBeLessThan(120000);
+  });
+
+  it("logs timing summary", () => {
+    const avg = Math.round(timingStats.total / 6);
+    console.log(`\n=== Experiment 52E Timing ===`);
+    console.log(`Calls: 6`);
+    console.log(`Total: ${timingStats.total}ms`);
+    console.log(`Average: ${avg}ms`);
+    console.log(`Fastest: ${timingStats.min}ms`);
+    console.log(`Slowest: ${timingStats.max}ms`);
+  });
+});