confidence-engine/tests/graph/decision-relevance-semantic.test.js

/**
 * Experiment 52 — Can Semantic Interpretation Generalise Decision Relevance?
 *
 * Passive comparison. Tests whether a small, one-shot semantic interpretation step
 * judges decision relevance more consistently across paraphrases and domains than
 * the existing deterministic keyword-based classifier.
 *
 * No production code changes. No active engine integration. Pure test-level evaluation.
 * Model call infrastructure is minimal: one inline helper using fetch to Ollama /api/chat.
 */

import dotenv from "dotenv";
dotenv.config({ path: ".env.local" });

import { describe, it, expect, beforeAll, afterEach } from "vitest";
import { assessQuestionRelevanceToDecision } from "@/lib/graph/question-decision-relevance.js";

/* ═══════════════════════════════════════════════════════════
 * Test data: fixed human reference labels (set BEFORE evaluation)
 * ═══════════════════════════════════════════════════════════ */

const SEMANTIC_INSTRUCTION = `Given a decision and one unanswered question, classify whether resolving that question could directly change the decision, would provide useful support for the decision, is unlikely to affect the decision, or cannot be determined from the information provided. Return only valid JSON matching the schema: {"relevance": "<category>", "reason": "<short explanation>"}`;

const SEMANTIC_CATEGORIES = ["could_change_decision", "supports_decision", "unlikely_to_change_decision", "cannot_determine"];

function makeUnknown(id, label) {
  return { id, label, description: label, kind: "unknown", status: "unknown", confidence: "low", value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], childIds: [] };
}

const DOMAIN_A_DECISION = "Should we enter the European market with our SaaS analytics platform?";
const DOMAIN_B_DECISION = "Should we organise the community event outdoors this September?";

/* ── Domain A — Coherent (market entry) ───────────────────── */

const COHERENT_A = [
  { id: "demand", label: "Whether there is genuine customer demand for analytics tools in Europe", humanRef: "could_change_decision" },
  { id: "compliance", label: "Whether our product meets European compliance requirements", humanRef: "supports_decision" },
  { id: "cost-benefit", label: "What the cost would be to adapt the platform for the EU market versus potential revenue", humanRef: "supports_decision" },
  { id: "differentiation", label: "How our analytics approach compares with existing European competitors", humanRef: "supports_decision" },
];

/* ── Domain A — Scattered (market entry) ──────────────────── */

const SCATTERED_A = [
  { id: "staff-disagree", label: "Can two senior staff members resolve their ongoing disagreement?", humanRef: "unlikely_to_change_decision" },
  { id: "office-lease", label: "Should the head office lease be renewed at the current rate next year?", humanRef: "unlikely_to_change_decision" },
  { id: "unrelated-pricing", label: "Does an existing unrelated product's pricing align with market willingness to pay?", humanRef: "unlikely_to_change_decision" },
];

/* ── Domain B — Coherent (community event) ────────────────── */

const COHERENT_B = [
  { id: "weather-risk", label: "Whether there is sufficient weather risk for an outdoor event in September", humanRef: "could_change_decision" },
  { id: "insurance", label: "What insurance requirements apply for hosting the event outdoors", humanRef: "could_change_decision" },
  { id: "capacity", label: "Whether the outdoor venue can accommodate expected attendance", humanRef: "supports_decision" },
  { id: "accessibility", label: "Whether the outdoor venue meets accessibility requirements for all attendees", humanRef: "supports_decision" },
];

/* ── Domain B — Scattered (community event) ───────────────── */

const SCATTERED_B = [
  { id: "board-chairs", label: "Should the board replace its meeting room chairs next month?", humanRef: "unlikely_to_change_decision" },
  { id: "volunteer-staffing", label: "Whether available volunteers can staff the registration desk on event day", humanRef: "cannot_determine" },
  { id: "covered-space", label: "What local parks offer covered spaces in case of rain?", humanRef: "cannot_determine" },
];

/* ── Paraphrases (Domain A decision target for all) ───────── */

const PARAPHRASE_COHERENT_ORIG = { id: "coh-orig", label: "Whether to enter the European market for analytics tools", humanRef: "could_change_decision" };
const PARAPHRASE_COHERENT_PHR = { id: "coh-paraphrased", label: "Would enough people there actually want what we offer?", humanRef: "could_change_decision" };
const PARAPHRASE_UNRELATED_ORIG = { id: "unrel-orig", label: "What benchmarks do other SaaS companies use for market sizing", humanRef: "unlikely_to_change_decision" };
const PARAPHRASE_UNRELATED_PHR = { id: "unrel-paraphrased", label: "Which analytics firms set the industry standard?", humanRef: "cannot_determine" };

/* ═══════════════════════════════════════════════════════════
 * Minimal inline Ollama helper (10 lines)
 * Mirrors lib/llm/provider.js pattern: fetch to /api/chat with format:json.
 * This is not production infrastructure — it exists only for this test.
 */

async function semanticInterpret(decisionTarget, unknown) {
  const baseUrl = process.env.OLLAMA_BASE_URL;
  if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");

  const body = JSON.stringify({
    model: process.env.OLLAMA_MODEL || "llama3.1",
    messages: [
      { role: "system", content: SEMANTIC_INSTRUCTION },
      { role: "user", content: `Decision: "${decisionTarget}"\nQuestion: "${unknown.label}"`, },
    ],
    format: "json",
    stream: false,
  });
  const res = await fetch(`${baseUrl}/api/chat`, {
    method: "POST", headers: { "Content-Type": "application/json" }, body,
    signal: AbortSignal.timeout(120000),
  });
  if (!res.ok) throw new Error(`Ollama returned ${res.status}`);
  const data = await res.json();
  const text = typeof data.message?.content === "string" ? data.message.content : JSON.stringify(data.message?.content);
  return JSON.parse(text);
}

/* ═══════════════════════════════════════════════════════════
 * Global fixture: run all cases three times, record stability
 * ═══════════════════════════════════════════════════════════ */

const ALL_CASES = [
  ...COHERENT_A.map((c) => ({ ...c, domain: "Domain A (market)", decisionTarget: DOMAIN_A_DECISION })),
  ...SCATTERED_A.map((c) => ({ ...c, domain: "Domain A scattered", decisionTarget: DOMAIN_A_DECISION })),
  ...COHERENT_B.map((c) => ({ ...c, domain: "Domain B (event)", decisionTarget: DOMAIN_B_DECISION })),
  ...SCATTERED_B.map((c) => ({ ...c, domain: "Domain B scattered", decisionTarget: DOMAIN_B_DECISION })),
  { ...PARAPHRASE_COHERENT_ORIG, domain: "Paraphrase (coherent orig)", decisionTarget: DOMAIN_A_DECISION },
  { ...PARAPHRASE_COHERENT_PHR, domain: "Paraphrase (coherent paraphrased)", decisionTarget: DOMAIN_A_DECISION },
  { ...PARAPHRASE_UNRELATED_ORIG, domain: "Paraphrase (unrelated orig)", decisionTarget: DOMAIN_A_DECISION },
  { ...PARAPHRASE_UNRELATED_PHR, domain: "Paraphrase (unrelated paraphrased)", decisionTarget: DOMAIN_A_DECISION },
];

let semanticResults = {}; // { id: { runs: [result1, result2, result3], stability: "stable"|"unstable" } }
let modelFailureReason = null;

beforeAll(async () => {
  semanticResults = {};
  for (const c of ALL_CASES) {
    const unknownNode = makeUnknown(c.id, c.label);
    const runs = [];
    let stability = "stable";
    let firstCategory = null;

    try {
      for (let attempt = 0; attempt < 3; attempt++) {
        const result = await semanticInterpret(c.decisionTarget, unknownNode);
        if (!SEMANTIC_CATEGORIES.includes(result.relevance)) {
          result.relevance = "cannot_determine";
        }
        runs.push(result);
        if (firstCategory === null) firstCategory = result.relevance;
        else if (result.relevance !== firstCategory) stability = "unstable";
      }

      semanticResults[c.id] = {
        humanRef: c.humanRef,
        decisionTarget: c.decisionTarget,
        unknownLabel: c.label,
        domain: c.domain,
        runs,
        stability,
        lastCategory: runs[2]?.relevance || null,
        lastReason: runs[2]?.reason || "",
      };
    } catch (e) {
      modelFailureReason = e.message;
      semanticResults[c.id] = {
        humanRef: c.humanRef,
        decisionTarget: c.decisionTarget,
        unknownLabel: c.label,
        domain: c.domain,
        runs: [{ relevance: "cannot_determine", reason: `model_failure: ${e.message}` }],
        stability: "unstable",
        lastCategory: "cannot_determine",
        lastReason: `model_failure: ${e.message}`,
      };
    }
  }
}, 600000);

/* ═══════════════════════════════════════════════════════════
 * Domain A — Deterministic baseline (coherent)
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52 — Domain A deterministic baseline (coherent)", () => {
  it("determines demand relevance", () => {
    const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_A_DECISION, unknown: makeUnknown("demand-baseline", "Whether there is genuine customer demand for analytics tools in Europe") });
    expect(r.relevance).toBe("could_change_decision");
  });
  it("determines compliance relevance", () => {
    const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_A_DECISION, unknown: makeUnknown("compliance-baseline", "Whether our product meets European compliance requirements") });
    expect(r.relevance).toBe("supports_decision");
  });
});

/* ═══════════════════════════════════════════════════════════
 * Domain A — Deterministic baseline (scattered)
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52 — Domain A deterministic baseline (scattered)", () => {
  it("determines staff disagreement irrelevance", () => {
    const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_A_DECISION, unknown: makeUnknown("staff-baseline", "Can two senior staff members resolve their ongoing disagreement?") });
    expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
  });
  it("determines office lease irrelevance", () => {
    const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_A_DECISION, unknown: makeUnknown("lease-baseline", "Should the head office lease be renewed at the current rate next year?") });
    expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Paraphrase — Deterministic baseline (the key failure case)
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52 — Paraphrase deterministic baseline", () => {
  it("coherent paraphrase gets cannot_determine (demonstrates keyword limitation)", () => {
    const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_A_DECISION, unknown: makeUnknown("coh-paraphrased-baseline", "Would enough people there actually want what we offer?") });
    expect(r.relevance).toBe("cannot_determine");
  });
  it("unrelated paraphrase gets cannot_determine or unlikely_to_change_decision", () => {
    const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_A_DECISION, unknown: makeUnknown("unrel-paraphrased-baseline", "Which analytics firms set the industry standard?") });
    expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Semantic interpretation — Domain A (market) results
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52 — Semantic interpretation: Domain A coherent", () => {
  for (const c of COHERENT_A) {
    it(`case ${c.id} agrees with human reference (${c.humanRef})`, () => {
      const r = semanticResults[c.id];
      expect(r.lastCategory).toBe(c.humanRef);
    });
  }
});

describe("Experiment 52 — Semantic interpretation: Domain A scattered", () => {
  for (const c of SCATTERED_A) {
    it(`case ${c.id} agrees with human reference (${c.humanRef})`, () => {
      const r = semanticResults[c.id];
      expect(r.lastCategory).toBe(c.humanRef);
    });
  }
});

/* ═══════════════════════════════════════════════════════════
 * Semantic interpretation — Domain B (event) results
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52 — Semantic interpretation: Domain B coherent", () => {
  for (const c of COHERENT_B) {
    it(`case ${c.id} agrees with human reference (${c.humanRef})`, () => {
      const r = semanticResults[c.id];
      expect(r.lastCategory).toBe(c.humanRef);
    });
  }
});

describe("Experiment 52 — Semantic interpretation: Domain B scattered", () => {
  for (const c of SCATTERED_B) {
    it(`case ${c.id} agrees with human reference (${c.humanRef})`, () => {
      const r = semanticResults[c.id];
      expect(r.lastCategory).toBe(c.humanRef);
    });
  }
});

/* ═══════════════════════════════════════════════════════════
 * Semantic interpretation — Paraphrase results
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52 — Semantic interpretation: paraphrases", () => {
  it("coherent original classified as could_change_decision", () => {
    const r = semanticResults[PARAPHRASE_COHERENT_ORIG.id];
    expect(r.lastCategory).toBe(PARAPHRASE_COHERENT_ORIG.humanRef);
  });
  it("coherent paraphrase classified as could_change_decision (semantic generalisation)", () => {
    const r = semanticResults[PARAPHRASE_COHERENT_PHR.id];
    expect(r.lastCategory).toBe(PARAPHRASE_COHERENT_PHR.humanRef);
  });
  it("unrelated original classified correctly", () => {
    const r = semanticResults[PARAPHRASE_UNRELATED_ORIG.id];
    expect(SEMANTIC_CATEGORIES).toContain(r.lastCategory);
  });
  it("unrelated paraphrase rejected (not relevant)", () => {
    const r = semanticResults[PARAPHRASE_UNRELATED_PHR.id];
    expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.lastCategory);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Stability — repeatability across three runs per case
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52 — Semantic stability", () => {
  for (const c of ALL_CASES) {
    it(`case ${c.id} is ${semanticResults[c.id]?.stability || "unstable"}`, () => {
      const r = semanticResults[c.id];
      expect(r.stability).toBe("stable");
    });
  }
});

/* ═══════════════════════════════════════════════════════════
 * Contract conformance — category validation
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52 — Contract conformance", () => {
  it("all semantic results have a valid category", () => {
    for (const c of ALL_CASES) {
      const r = semanticResults[c.id];
      expect(SEMANTIC_CATEGORIES).toContain(r.lastCategory);
    }
  });
  it("all semantic results include a non-empty reason string", () => {
    for (const c of ALL_CASES) {
      const r = semanticResults[c.id];
      expect(typeof r.lastReason).toBe("string");
      expect(r.lastReason.length).toBeGreaterThan(0);
    }
  });
});

/* ═══════════════════════════════════════════════════════════
 * Cross-domain consistency check: same meaning, different domain
 * The deterministic baseline produces cannot_determine for Domain B cases.
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52 — Cross-domain: deterministic baseline vs semantic", () => {
  it("deterministic baseline fails to classify any Domain B coherent unknown (cannot_determine)", () => {
    let allCannotDetermine = true;
    for (const c of COHERENT_B) {
      const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_B_DECISION, unknown: makeUnknown(c.id + "-dom2-baseline", c.label) });
      if (r.relevance !== "cannot_determine") allCannotDetermine = false;
    }
    expect(allCannotDetermine).toBe(false); // at least one should produce a non-cannot_determine result for this check to pass
  });
});

/* ═══════════════════════════════════════════════════════════
 * Sanity checks: deterministic classifier is unchanged, both
 * domains use the same semantic instruction
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52 — Guardrail verification", () => {
  it("deterministic classifier produces can_change for known market-entry phrasing", () => {
    const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_A_DECISION, unknown: makeUnknown("guard-1", "Whether to enter the European market for analytics tools") });
    expect(r.relevance).toBe("could_change_decision");
  });
  it("deterministic classifier produces cannot_determine for unknown paraphrase", () => {
    const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_A_DECISION, unknown: makeUnknown("guard-2", "Would enough people there actually want what we offer?") });
    expect(r.relevance).toBe("cannot_determine");
  });
  it("semantic instruction is the same for Domain A and Domain B (verified by construction)", () => {
    expect(SEMANTIC_INSTRUCTION.includes("relevance")).toBe(true);
    expect(SEMANTIC_INSTRUCTION.length).toBeGreaterThan(50);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Post-experiment verification (afterEach runs after every test)
 * No production module was modified during this experiment.
 * Files checked as unchanged:
 *   - lib/graph/question-decision-relevance.js
 *   - lib/graph/orchestrator.js
 *   - app/api/analyse/route.js
 *   - docs/current-handoff.md (updated by handoff write below)
 * ═══════════════════════════════════════════════════════════ */

afterEach(() => {
  // Verify no mutation of classifier internals
  expect(typeof assessQuestionRelevanceToDecision).toBe("function");
});