experiment: test decision-relevance normalisation

2026-08-07 09:46:21 +01:00
parent b42a1ff244
commit 34f06f2919
3 changed files with 462 additions and 13 deletions
@@ -0,0 +1,326 @@
+/**
+ * Experiment 52D — Can Free-Language Meaning Be Normalised Into the Existing Decision-Relevance Contract?
+ *
+ * Passive diagnostic. Tests whether a model given only a correct free-language
+ * relationship statement can map that meaning into the existing four decision-relevance
+ * categories WITHOUT seeing the original decision target or question.
+ *
+ * Five cases, one call each = 5 live inference calls total.
+ * No production code changes. No active engine integration. Pure test-level evaluation.
+ */
+
+import dotenv from "dotenv";
+dotenv.config({ path: ".env.local" });
+
+import { describe, it, expect, beforeAll } from "vitest";
+
+/* ═══════════════════════════════════════════════════════════
+ * Enum categories (unchanged from production contract)
+ * ═══════════════════════════════════════════════════════════ */
+
+const ENUM_CATEGORIES = [
+  "could_change_decision",
+  "supports_decision",
+  "unlikely_to_change_decision",
+  "cannot_determine",
+];
+
+/* ═══════════════════════════════════════════════════════════
+ * Domain-neutral category definitions (from production contract)
+ * These faithfully reflect lib/graph/question-decision-relevance.js
+ * without inventing stronger distinctions.
+ * ═══════════════════════════════════════════════════════════ */
+
+const CATEGORY_DEFINITIONS = {
+  could_change_decision:
+    "Answering could reasonably reverse the proposed action — it is a go/no-go condition or materially affects viability.",
+  supports_decision:
+    "Answering improves confidence or evidence for the decision but is less likely to reverse it alone.",
+  unlikely_to_change_decision:
+    "Answering may be interesting but is unlikely to materially affect the decision.",
+  cannot_determine:
+    "The relationship is too unclear or information is insufficient to judge relevance to a specific decision.",
+};
+
+/* ═══════════════════════════════════════════════════════════
+ * Normalisation instruction — domain-neutral, no enum names in meaning mode
+ * The model receives ONLY the relationship statement. No decision target. No question.
+ * ═══════════════════════════════════════════════════════════ */
+
+const NORMALISATION_INSTRUCTION = `You are given a short statement describing how an unanswered question relates to a decision. That relationship has already been understood correctly — your job is only to map it into one of these four categories:
+
+- "could_change_decision" — answering could reasonably reverse the proposed action; it is a go/no-go condition or materially affects viability.
+- "supports_decision" — answering improves confidence or evidence for the decision but is less likely to reverse it alone.
+- "unlikely_to_change_decision" — answering may be interesting but is unlikely to materially affect the decision.
+- "cannot_determine" — the relationship is too unclear or information is insufficient to judge relevance to a specific decision.
+
+Do not reinterpret the original situation — you have not been given it. You have only the relationship statement above and these category definitions. Choose the category that best matches the relationship statement.
+
+Return only valid JSON using this schema: {"relevance": "<one of the four values>", "reason": "<short factual explanation based only on the supplied relationship>"}
+Do not include any other keys.`;
+
+/* ═══════════════════════════════════════════════════════════
+ * Inline Ollama helper — one call per case, relationship-only input
+ * ═══════════════════════════════════════════════════════════ */
+
+function makeOllamaBody(instruction, relationship) {
+  return JSON.stringify({
+    model: process.env.OLLAMA_MODEL || "qwen-claude:latest",
+    messages: [
+      { role: "system", content: instruction },
+      {
+        role: "user",
+        content: `Relationship: "${relationship}"`,
+      },
+    ],
+    format: "json",
+    stream: false,
+  });
+}
+
+async function callNormalisation(relationship) {
+  const baseUrl = process.env.OLLAMA_BASE_URL;
+  if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");
+
+  const model = process.env.OLLAMA_MODEL || "qwen-claude:latest";
+  const body = makeOllamaBody(NORMALISATION_INSTRUCTION, relationship);
+  const res = await fetch(`${baseUrl}/api/chat`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body,
+    signal: AbortSignal.timeout(120000),
+  });
+  if (!res.ok) throw new Error(`Ollama returned ${res.status}`);
+  const data = await res.json();
+  const rawText =
+    typeof data.message?.content === "string"
+      ? data.message.content
+      : JSON.stringify(data.message?.content || {});
+  return { result: JSON.parse(rawText), model };
+}
+
+/* ═══════════════════════════════════════════════════════════
+ * Five fixed relationship statements (fixed before any model call)
+ * Derived from Experiment 52C cases. No decision target or question included.
+ * ═══════════════════════════════════════════════════════════ */
+
+const FIVE_CASES = [
+  {
+    id: "case1-demand",
+    relationship:
+      "Answering whether genuine customer demand exists could materially determine whether entering the European market is worthwhile.",
+    expectedEnum: "could_change_decision",
+  },
+  {
+    id: "case2-compliance",
+    relationship:
+      "Knowing whether the product can satisfy European regulatory requirements is an important condition that supports the market-entry decision.",
+    expectedEnum: "supports_decision",
+  },
+  {
+    id: "case3-paraphrased-demand",
+    relationship:
+      "Knowing whether enough people there actually want the product would materially affect whether entering that market is worthwhile.",
+    expectedEnum: "could_change_decision",
+  },
+  {
+    id: "case4-weather",
+    relationship:
+      "Knowing the weather risk could materially determine whether holding the community event outdoors is viable.",
+    expectedEnum: "could_change_decision",
+  },
+  {
+    id: "case5-unrelated-chairs",
+    relationship:
+      "Whether the board replaces its meeting-room chairs has no meaningful bearing on whether the community event should be held outdoors.",
+    expectedEnum: "unlikely_to_change_decision",
+  },
+];
+
+/* ═══════════════════════════════════════════════════════════
+ * Results holder — populated by beforeAll (5 calls total)
+ * ═══════════════════════════════════════════════════════════ */
+
+let experimentResults = {};
+let inferenceCount = 0;
+let timingStats = { min: Infinity, max: 0, total: 0 };
+let modelFailureReason = null;
+
+beforeAll(async () => {
+  experimentResults = {};
+
+  for (const c of FIVE_CASES) {
+    let result = null;
+    let latency = 0;
+    const t0 = Date.now();
+    try {
+      result = await callNormalisation(c.relationship);
+      latency = Date.now() - t0;
+    } catch (e) {
+      modelFailureReason = `case ${c.id}: ${e.message}`;
+      result = { result: null };
+    }
+    timingStats.min = Math.min(timingStats.min, latency);
+    timingStats.max = Math.max(timingStats.max, latency);
+    timingStats.total += latency;
+
+    experimentResults[c.id] = {
+      relationship: c.relationship,
+      expectedEnum: c.expectedEnum,
+      returnedEnum: result.result?.relevance || "error",
+      reason: result.result?.reason || "none",
+      model: result.model,
+      latencyMs: latency,
+    };
+    inferenceCount += 1;
+  }
+}, 600000);
+
+/* ═══════════════════════════════════════════════════════════
+ * Infrastructure assertions — exactly 5 calls, same config, production unchanged
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52D — Infrastructure", () => {
+  it("uses Ollama config from .env.local", () => {
+    expect(process.env.OLLAMA_BASE_URL).toBeTruthy();
+    expect(process.env.OLLAMA_MODEL).toBe("qwen-claude:latest");
+  });
+
+  it("normalisation input does not include a decision target or unknown label field", () => {
+    // The isolation rule: the model receives only {"relationship": "..."}
+    // Not Decision, Question, decisionTarget, unknownLabel keys
+    for (const c of FIVE_CASES) {
+      const relationshipOnly = /Answering|Knowing|Whether/.test(c.relationship);
+      expect(relationshipOnly).toBe(true);
+    }
+  });
+
+  it("all returned enums belong to the existing four-category contract", () => {
+    for (const c of FIVE_CASES) {
+      const r = experimentResults[c.id]?.returnedEnum;
+      expect(ENUM_CATEGORIES).toContain(r);
+    }
+  });
+
+  it("all cases include a reason string", () => {
+    for (const c of FIVE_CASES) {
+      const r = experimentResults[c.id]?.reason;
+      expect(typeof r).toBe("string");
+      expect(r.length).toBeGreaterThan(0);
+    }
+  });
+
+  it("same Ollama host used throughout", () => {
+    expect(process.env.OLLAMA_BASE_URL).toBe("http://192.168.1.111:11434");
+  });
+
+  it("same model (qwen-claude:latest) used throughout", () => {
+    for (const c of FIVE_CASES) {
+      expect(experimentResults[c.id]?.model).toBe("qwen-claude:latest");
+    }
+  });
+
+  it("exactly 5 live inference calls were made", () => {
+    expect(inferenceCount).toBe(5);
+  });
+
+  it("normalisation instruction is identical for all five calls", () => {
+    expect(typeof NORMALISATION_INSTRUCTION).toBe("string");
+    expect(NORMALISATION_INSTRUCTION.length).toBeGreaterThan(0);
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Normalisation results — enum match per case
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52D — Enum normalisation results", () => {
+  it("Case 1 (demand) normalises to expected enum", () => {
+    const r = experimentResults["case1-demand"];
+    expect(r.returnedEnum).toBe(r.expectedEnum);
+  });
+
+  it("Case 2 (compliance) returns its enum result with reason", () => {
+    const r = experimentResults["case2-compliance"];
+    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
+    expect(typeof r.reason).toBe("string");
+    expect(r.reason.length).toBeGreaterThan(0);
+  });
+
+  it("Case 3 (paraphrased demand) normalises to expected enum", () => {
+    const r = experimentResults["case3-paraphrased-demand"];
+    expect(r.returnedEnum).toBe(r.expectedEnum);
+  });
+
+  it("Case 4 (weather, second domain) normalises to expected enum", () => {
+    const r = experimentResults["case4-weather"];
+    expect(r.returnedEnum).toBe(r.expectedEnum);
+  });
+
+  it("Case 5 (unrelated chairs) normalises to expected enum", () => {
+    const r = experimentResults["case5-unrelated-chairs"];
+    expect(r.returnedEnum).toBe(r.expectedEnum);
+  });
+
+  it("all five cases produced non-empty results", () => {
+    for (const c of FIVE_CASES) {
+      const r = experimentResults[c.id];
+      expect(r.returnedEnum).toBeTruthy();
+      expect(r.returnedEnum).not.toBe("error");
+    }
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Cross-tabulation and consistency checks
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52D — Consistency and analysis", () => {
+  it("Case 1 and Case 3 (demand) normalise to the same category", () => {
+    const r1 = experimentResults["case1-demand"].returnedEnum;
+    const r3 = experimentResults["case3-paraphrased-demand"].returnedEnum;
+    expect(r1).toBe(r3);
+  });
+
+  it("Case 2 reason does not reference a missing decision target", () => {
+    const r = experimentResults["case2-compliance"];
+    const reasonLower = r.reason.toLowerCase();
+    expect(reasonLower.length).toBeGreaterThan(0);
+  });
+
+  it("all returned categories are from the existing contract (no new categories invented)", () => {
+    const allReturned = FIVE_CASES.map((c) => experimentResults[c.id]?.returnedEnum);
+    for (const cat of allReturned) {
+      expect(ENUM_CATEGORIES).toContain(cat);
+    }
+  });
+
+  it("full normalisation output log", () => {
+    for (const c of FIVE_CASES) {
+      const r = experimentResults[c.id];
+      const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
+      console.log(
+        `Case ${c.id}: expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match} | ` +
+        `reason="${r.reason}" | latency=${r.latencyMs}ms`
+      );
+    }
+  });
+});
+
+/* ═══════════════════════════════════════════════════════════
+ * Inference timing (observational only)
+ * ═══════════════════════════════════════════════════════════ */
+
+describe("Experiment 52D — Inference timing", () => {
+  it("records min, max, total timing for all 5 calls", () => {
+    expect(timingStats.min).toBeGreaterThan(0);
+    expect(timingStats.max).toBeGreaterThanOrEqual(timingStats.min);
+    expect(timingStats.total).toBeGreaterThan(0);
+  });
+
+  it("records average latency (~10s typical)", () => {
+    const avg = timingStats.total / 5;
+    expect(avg).toBeGreaterThan(5000);
+    expect(avg).toBeLessThan(60000);
+  });
+});