confidence-engine/tests/graph/decision-relevance-ambiguity.test.js

/**
 * Experiment 52F — Will the Normaliser Admit When the Category Boundary Is Genuinely Unclear?
 *
 * Passive contract-boundary experiment. Tests whether the existing decision-relevance normalisation
 * contract can preserve ambiguity instead of forcing an unclear relationship into a stronger category.
 *
 * Four fixed relationship statements. Four live inference calls. One per case.
 * Uses exactly the same category definitions and normalisation instruction as Experiment 52E.
 * Does not change any production code, category definitions, classifier, or active engine.
 */

import dotenv from "dotenv";
dotenv.config({ path: ".env.local" });

import { describe, it, expect, beforeAll } from "vitest";

/* ═══════════════════════════════════════════════════════════
 * Enum categories (unchanged from production contract)
 * ═══════════════════════════════════════════════════════════ */

const ENUM_CATEGORIES = [
  "could_change_decision",
  "supports_decision",
  "unlikely_to_change_decision",
  "cannot_determine",
];

/* ═══════════════════════════════════════════════════════════
 * Category definitions — identical to Experiment 52E and production
 * ═══════════════════════════════════════════════════════════ */

const CATEGORY_DEFINITIONS = {
  could_change_decision:
    "Answering could reasonably reverse the proposed action — it is a go/no-go condition or materially affects viability.",
  supports_decision:
    "Answering improves confidence or evidence for the decision but is less likely to reverse it alone.",
  unlikely_to_change_decision:
    "Answering may be interesting but is unlikely to materially affect the decision.",
  cannot_determine:
    "The relationship is too unclear or information is insufficient to judge relevance to a specific decision.",
};

/* ═══════════════════════════════════════════════════════════
 * Normalisation instruction — identical to Experiment 52E
 * ═══════════════════════════════════════════════════════════ */

const NORMALISATION_INSTRUCTION = `You are given a short statement describing how an unanswered question relates to a decision. That relationship has already been understood correctly — your job is only to map it into one of these four categories:

- "could_change_decision" — answering could reasonably reverse the proposed action; it is a go/no-go condition or materially affects viability.
- "supports_decision" — answering improves confidence or evidence for the decision but is less likely to reverse it alone.
- "unlikely_to_change_decision" — answering may be interesting but is unlikely to materially affect the decision.
- "cannot_determine" — the relationship is too unclear or information is insufficient to judge relevance to a specific decision.

Do not reinterpret the original situation — you have not been given it. You have only the relationship statement above and these category definitions. Choose the category that best matches the relationship statement.

Return only valid JSON using this schema: {"relevance": "<one of the four values>", "reason": "<short factual explanation based only on the supplied relationship>"}
Do not include any other keys.`;

/* ═══════════════════════════════════════════════════════════
 * Inline Ollama helper — one call per case, relationship-only input
 * ═══════════════════════════════════════════════════════════ */

function makeOllamaBody(instruction, relationship) {
  return JSON.stringify({
    model: process.env.OLLAMA_MODEL || "qwen-claude:latest",
    messages: [
      { role: "system", content: instruction },
      {
        role: "user",
        content: `Relationship: "${relationship}"`,
      },
    ],
    format: "json",
    stream: false,
  });
}

async function callAmbiguityTest(relationship) {
  const baseUrl = process.env.OLLAMA_BASE_URL;
  if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");

  const model = process.env.OLLAMA_MODEL || "qwen-claude:latest";
  const body = makeOllamaBody(NORMALISATION_INSTRUCTION, relationship);
  const res = await fetch(`${baseUrl}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body,
    signal: AbortSignal.timeout(120000),
  });
  if (!res.ok) throw new Error(`Ollama returned ${res.status}`);
  const data = await res.json();
  const rawText =
    typeof data.message?.content === "string"
      ? data.message.content
      : JSON.stringify(data.message?.content || {});
  return { result: JSON.parse(rawText), model };
}

/* ═══════════════════════════════════════════════════════════
 * Four fixed relationship statements (fixed before any model call)
 * ═══════════════════════════════════════════════════════════ */

const CASES = [
  {
    id: "case1-clear-blocker",
    description: "Clear blocker control",
    relationship:
      "If the product cannot satisfy the required regulations, entering the market cannot legally proceed.",
    expectedEnum: "could_change_decision",
  },
  {
    id: "case2-clear-support",
    description: "Clear support control",
    relationship:
      "Evidence that the product already meets commonly expected regulatory standards would increase confidence in entering the market, but would not determine the decision by itself.",
    expectedEnum: "supports_decision",
  },
  {
    id: "case3-ambiguous",
    description: "Genuinely ambiguous",
    relationship:
      "Understanding the regulatory position would be important to the market-entry decision.",
    expectedEnum: "cannot_determine",
  },
  {
    id: "case4-ambiguous-condition",
    description: "Ambiguous condition",
    relationship:
      "Regulatory compliance is an important condition to consider when deciding whether to enter the market.",
    expectedEnum: "cannot_determine",
  },
];

/* ═══════════════════════════════════════════════════════════
 * External-assumption diagnostic helper
 * ═══════════════════════════════════════════════════════════ */

function checkGrounding(expectedEnum, reason, relationship) {
  if (expectedEnum !== "cannot_determine") return "grounded_only_in_statement";

  const reasonLower = reason.toLowerCase();
  // Check if the model introduces external assumptions about regulation being a blocker
  // rather than reasoning from the supplied statement alone
  const externalAssumptionSignals = [
    /regulatory.*always|compliance.*always|regulation.*must.*block|regulation.*mandatory.*requirement/i,
    /legal.*prerequisite|cannot proceed without|strictly required|legally mandatory/i,
    /by definition.*regul|inherently.*blocking|necessarily.*prevent/i,
  ];

  const hasExternalSignal = externalAssumptionSignals.some((p) => p.test(reasonLower));

  // Check if the model grounds its reasoning only in what the statement actually says
  // vs importing outside knowledge that regulation is always a blocker
  return hasExternalSignal ? "introduced_external_assumption" : "grounded_only_in_statement";
}

/* ═══════════════════════════════════════════════════════════
 * Results holder — populated by beforeAll (4 calls total)
 * ═══════════════════════════════════════════════════════════ */

let experimentResults = {};
let inferenceCount = 0;
let timingStats = { min: Infinity, max: 0, total: 0 };
let modelFailureReason = null;

beforeAll(async () => {
  experimentResults = {};

  for (const c of CASES) {
    let result = null;
    let latency = 0;
    const t0 = Date.now();
    try {
      result = await callAmbiguityTest(c.relationship);
      latency = Date.now() - t0;
    } catch (e) {
      modelFailureReason = `case ${c.id}: ${e.message}`;
      result = { result: null };
    }
    timingStats.min = Math.min(timingStats.min, latency);
    timingStats.max = Math.max(timingStats.max, latency);
    timingStats.total += latency;

    experimentResults[c.id] = {
      description: c.description,
      relationship: c.relationship,
      expectedEnum: c.expectedEnum,
      returnedEnum: result.result?.relevance || "error",
      reason: result.result?.reason || "none",
      model: result.model,
      latencyMs: latency,
    };
    inferenceCount += 1;
  }
}, 600000);

/* ═══════════════════════════════════════════════════════════
 * Infrastructure assertions — exactly 4 calls, same config, production unchanged
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52F — Infrastructure", () => {
  it("uses Ollama config from .env.local", () => {
    expect(process.env.OLLAMA_BASE_URL).toBeTruthy();
    expect(process.env.OLLAMA_MODEL).toBe("qwen-claude:latest");
  });

  it("all returned enums belong to the existing four-category contract", () => {
    for (const c of CASES) {
      const r = experimentResults[c.id]?.returnedEnum;
      expect(ENUM_CATEGORIES).toContain(r);
    }
  });

  it("all cases include a reason string", () => {
    for (const c of CASES) {
      const r = experimentResults[c.id]?.reason;
      expect(typeof r).toBe("string");
      expect(r.length).toBeGreaterThan(0);
    }
  });

  it("same Ollama host used throughout", () => {
    expect(process.env.OLLAMA_BASE_URL).toBe("http://192.168.1.111:11434");
  });

  it("same model (qwen-claude:latest) used throughout", () => {
    for (const c of CASES) {
      expect(experimentResults[c.id]?.model).toBe("qwen-claude:latest");
    }
  });

  it("exactly 4 live inference calls were made", () => {
    expect(inferenceCount).toBe(4);
  });

  it("normalisation instruction is identical for all four calls", () => {
    expect(typeof NORMALISATION_INSTRUCTION).toBe("string");
    expect(NORMALISATION_INSTRUCTION.length).toBeGreaterThan(0);
  });

  it("fixed expected enums before any live call — structure check", () => {
    let blockerCount = 0;
    let supportingCount = 0;
    let ambiguousCount = 0;
    for (const c of CASES) {
      if (c.expectedEnum === "could_change_decision") blockerCount++;
      if (c.expectedEnum === "supports_decision") supportingCount++;
      if (c.expectedEnum === "cannot_determine") ambiguousCount++;
    }
    expect(blockerCount).toBe(1);
    expect(supportingCount).toBe(1);
    expect(ambiguousCount).toBe(2);
  });

  it("only the relationship statement is supplied to each case", () => {
    for (const c of CASES) {
      expect(c.relationship).toBeTruthy();
      expect(typeof c.relationship).toBe("string");
      expect(c).not.toHaveProperty("decisionTarget");
      expect(c).not.toHaveProperty("question");
    }
  });

  it("category definitions unchanged from production", () => {
    const expectedKeys = Object.keys(CATEGORY_DEFINITIONS);
    expect(expectedKeys).toContain("could_change_decision");
    expect(expectedKeys).toContain("supports_decision");
    expect(expectedKeys).toContain("unlikely_to_change_decision");
    expect(expectedKeys).toContain("cannot_determine");

    expect(CATEGORY_DEFINITIONS.could_change_decision).toMatch(/go\/no-go|materially affects viability/i);
    expect(CATEGORY_DEFINITIONS.supports_decision).toMatch(/improves confidence|less likely to reverse/i);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Case 1 — Clear blocker control
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52F — Case 1: Clear blocker", () => {
  it("maps to could_change_decision with match", () => {
    const r = experimentResults["case1-clear-blocker"];
    expect(r.returnedEnum).toBe(r.expectedEnum);
    expect(typeof r.reason).toBe("string");
    expect(r.reason.length).toBeGreaterThan(0);
  });

  it("grounds reasoning in the supplied statement only", () => {
    const r = experimentResults["case1-clear-blocker"];
    const reasonLower = r.reason.toLowerCase();
    // The relationship says "cannot legally proceed" — the model should reference that, not invent external regulation facts
    expect(reasonLower).toMatch(/legally.*proceed|cannot.*proceed|blocker|go\/no-go|must.*satisfy/);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Case 2 — Clear support control
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52F — Case 2: Clear supporting evidence", () => {
  it("maps to supports_decision with match", () => {
    const r = experimentResults["case2-clear-support"];
    expect(r.returnedEnum).toBe(r.expectedEnum);
    expect(typeof r.reason).toBe("string");
    expect(r.reason.length).toBeGreaterThan(0);
  });

  it("grounds reasoning in the supplied statement only", () => {
    const r = experimentResults["case2-clear-support"];
    const reasonLower = r.reason.toLowerCase();
    // The relationship explicitly says "increase confidence" and "would not determine"
    expect(reasonLower).toMatch(/confidence|evidence|improves.*support|not.*determine/);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Case 3 — Genuinely ambiguous
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52F — Case 3: Ambiguous relationship", () => {
  it("returns its enum result", () => {
    const r = experimentResults["case3-ambiguous"];
    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
    expect(typeof r.reason).toBe("string");
    expect(r.reason.length).toBeGreaterThan(0);
  });

  it("expected enum is cannot_determine", () => {
    const r = experimentResults["case3-ambiguous"];
    expect(r.expectedEnum).toBe("cannot_determine");
  });

  it("grounding check for case 3", () => {
    const r = experimentResults["case3-ambiguous"];
    const grounding = checkGrounding(r.expectedEnum, r.reason, r.relationship);
    expect(grounding).toBeTruthy();
    // Log the result
    console.log(`Case 3 grounding: ${grounding}`);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Case 4 — Ambiguous condition
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52F — Case 4: Ambiguous condition", () => {
  it("returns its enum result", () => {
    const r = experimentResults["case4-ambiguous-condition"];
    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
    expect(typeof r.reason).toBe("string");
    expect(r.reason.length).toBeGreaterThan(0);
  });

  it("expected enum is cannot_determine", () => {
    const r = experimentResults["case4-ambiguous-condition"];
    expect(r.expectedEnum).toBe("cannot_determine");
  });

  it("grounding check for case 4", () => {
    const r = experimentResults["case4-ambiguous-condition"];
    const grounding = checkGrounding(r.expectedEnum, r.reason, r.relationship);
    expect(grounding).toBeTruthy();
    // Log the result
    console.log(`Case 4 grounding: ${grounding}`);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Overall results — match assessment
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52F — Overall results", () => {
  it("reports all matches and mismatches", () => {
    for (const c of CASES) {
      const r = experimentResults[c.id];
      const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
      console.log(
        `[${c.description}] ${c.id}: expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match} | ` +
          `reason="${r.reason}" | latency=${r.latencyMs}ms`
      );
    }
  });

  it("clear controls (cases 1 & 2) both match", () => {
    const r1 = experimentResults["case1-clear-blocker"];
    const r2 = experimentResults["case2-clear-support"];
    expect(r1.returnedEnum).toBe(r1.expectedEnum);
    expect(r2.returnedEnum).toBe(r2.expectedEnum);
  });

  it("ambiguous cases (3 & 4) evaluate to cannot_determine", () => {
    const r3 = experimentResults["case3-ambiguous"];
    const r4 = experimentResults["case4-ambiguous-condition"];
    // These assertions will fail if the model forces them into stronger categories
    // — that failure is itself a finding we want to surface
    expect(r3.returnedEnum).toBe("cannot_determine");
    expect(r4.returnedEnum).toBe("cannot_determine");
  });

  it("clear-control match count", () => {
    const r1 = experimentResults["case1-clear-blocker"];
    const r2 = experimentResults["case2-clear-support"];
    let clearMatchCount = 0;
    if (r1.returnedEnum === r1.expectedEnum) clearMatchCount++;
    if (r2.returnedEnum === r2.expectedEnum) clearMatchCount++;
    expect(clearMatchCount).toBe(2);
  });

  it("cannot_determine count for ambiguous cases", () => {
    const r3 = experimentResults["case3-ambiguous"];
    const r4 = experimentResults["case4-ambiguous-condition"];
    let canNotDetermineCount = 0;
    if (r3.returnedEnum === "cannot_determine") canNotDetermineCount++;
    if (r4.returnedEnum === "cannot_determine") canNotDetermineCount++;
    expect(canNotDetermineCount).toBe(2);
  });

  it("external-assumption diagnostic for ambiguous cases", () => {
    const r3 = experimentResults["case3-ambiguous"];
    const r4 = experimentResults["case4-ambiguous-condition"];

    const g3 = checkGrounding(r3.expectedEnum, r3.reason, r3.relationship);
    const g4 = checkGrounding(r4.expectedEnum, r4.reason, r4.relationship);

    console.log(`External assumption Case 3: ${g3}`);
    console.log(`External assumption Case 4: ${g4}`);

    // Document whether either case introduced external assumptions
    const introducedAssumptions = [g3, g4].filter((g) => g === "introduced_external_assumption").length;
    // We document but don't assert — the finding is informational
    expect(typeof introducedAssumptions).toBe("number");
  });

  it("full output log", () => {
    console.log("\n=== Experiment 52F Summary ===");
    for (const c of CASES) {
      const r = experimentResults[c.id];
      const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
      const grounding = checkGrounding(r.expectedEnum, r.reason, r.relationship);
      console.log(
        `[${c.description}] ${r.expectedEnum} → ${r.returnedEnum} (${match}) | ` +
          `reason="${r.reason}" | grounding=${grounding} | latency=${r.latencyMs}ms`
      );
    }
  });
});

/* ═══════════════════════════════════════════════════════════
 * Inference timing (observational only)
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52F — Inference timing", () => {
  it("records min, max, total timing for all 4 calls", () => {
    expect(timingStats.min).toBeGreaterThan(0);
    expect(timingStats.max).toBeGreaterThanOrEqual(timingStats.min);
    expect(timingStats.total).toBeGreaterThan(0);
  });

  it("records average latency within reasonable range", () => {
    const avg = timingStats.total / 4;
    expect(avg).toBeGreaterThan(5000);
    expect(avg).toBeLessThan(120000);
  });

  it("logs timing summary", () => {
    const avg = Math.round(timingStats.total / 4);
    console.log(`\n=== Experiment 52F Timing ===`);
    console.log(`Calls: 4`);
    console.log(`Total: ${timingStats.total}ms`);
    console.log(`Average: ${avg}ms`);
    console.log(`Fastest: ${timingStats.min}ms`);
    console.log(`Slowest: ${timingStats.max}ms`);
  });
});