confidence-engine/tests/graph/decision-relevance-semantic.test.js

/**
 * Experiment 52B — Small Semantic Probe With the Existing Qwen Model
 *
 * Passive comparison. Tests whether a small, one-shot semantic interpretation step
 * judges decision relevance more consistently across paraphrases and domains than
 * the existing deterministic keyword-based classifier.
 *
 * Six cases from Experiment 52 corpus, run exactly once each.
 * No production code changes. No active engine integration. Pure test-level evaluation.
 * Same configured host and model as Experiment 52/52A.
 */

import dotenv from "dotenv";
dotenv.config({ path: ".env.local" });

import { describe, it, expect, beforeAll } from "vitest";
import { assessQuestionRelevanceToDecision } from "@/lib/graph/question-decision-relevance.js";

/* ═══════════════════════════════════════════════════════════
 * Semantic instruction — unchanged concept from Experiment 52.
 * Added explicit category enum so the model outputs our schema
 * (qwen-claude:latest needs this to avoid natural-language labels).
 * ═══════════════════════════════════════════════════════════ */

const SEMANTIC_INSTRUCTION = `Given a decision and one unanswered question, classify whether resolving that question could directly change the decision, would provide useful support for the decision, is unlikely to affect the decision, or cannot be determined from the information provided.

Return only valid JSON using exactly these category values (no others):
- "could_change_decision" — answering could reasonably reverse the proposed action
- "supports_decision" — answering improves confidence/evidence but less likely to reverse alone
- "unlikely_to_change_decision" — answering is unlikely to materially affect the decision
- "cannot_determine" — information is insufficient to judge

Schema: {"relevance": "<one of the four values above>", "reason": "<short explanation>"}

Do not use other words like "high", "low", "direct", etc. Use only the four category names listed.`;

const SEMANTIC_CATEGORIES = ["could_change_decision", "supports_decision", "unlikely_to_change_decision", "cannot_determine"];

/* ═══════════════════════════════════════════════════════════
 * Minimal inline Ollama helper — mirrors production pattern
 * ═══════════════════════════════════════════════════════════ */

async function semanticInterpret(decisionTarget, unknownLabel) {
  const baseUrl = process.env.OLLAMA_BASE_URL;
  if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");

  const model = process.env.OLLAMA_MODEL || "llama3.1";

  const body = JSON.stringify({
    model,
    messages: [
      { role: "system", content: SEMANTIC_INSTRUCTION },
      { role: "user", content: `Decision: "${decisionTarget}"\nQuestion: "${unknownLabel}"`, },
    ],
    format: "json",
    stream: false,
  });
  const res = await fetch(`${baseUrl}/api/chat`, {
    method: "POST", headers: { "Content-Type": "application/json" }, body,
    signal: AbortSignal.timeout(120000),
  });
  if (!res.ok) throw new Error(`Ollama returned ${res.status}`);
  const data = await res.json();
  const rawText = typeof data.message?.content === "string" ? data.message.content : JSON.stringify(data.message?.content || {});
  return { result: JSON.parse(rawText), model, latencyMs: 0 };
}

/* ═══════════════════════════════════════════════════════════
 * Decision targets (from Experiment 52)
 * ═══════════════════════════════════════════════════════════ */

const DOMAIN_A_DECISION = "Should we enter the European market with our SaaS analytics platform?";
const DOMAIN_B_DECISION = "Should we organise the community event outdoors this September?";

/* ═══════════════════════════════════════════════════════════
 * Experiment 52B — Six live inference cases (exactly 6 calls)
 *
 * Cases drawn from the existing Experiment 52 corpus.
 * Human reference labels fixed before evaluation; not changed after seeing results.
 * ═══════════════════════════════════════════════════════════ */

const SIX_CASES = [
  {
    id: "case1-familiar-relevant",
    decisionTarget: DOMAIN_A_DECISION,
    unknownLabel: "Whether there is genuine customer demand for analytics tools in Europe",
    humanRef: "could_change_decision",
    purpose: "Confirm semantic interpretation handles an easy in-domain relevant question.",
  },
  {
    id: "case2-familiar-unrelated",
    decisionTarget: DOMAIN_A_DECISION,
    unknownLabel: "Can two senior staff members resolve their ongoing disagreement?",
    humanRef: "unlikely_to_change_decision",
    purpose: "Confirm semantic interpretation can reject an obviously unrelated question.",
  },
  {
    id: "case3-paraphrase-relevant",
    decisionTarget: DOMAIN_A_DECISION,
    unknownLabel: "Would enough people there actually want what we offer?",
    humanRef: "could_change_decision",
    purpose: "Known deterministic keyword failure — semantic model should handle this paraphrase.",
  },
  {
    id: "case4-domain-b-relevant",
    decisionTarget: DOMAIN_B_DECISION,
    unknownLabel: "Whether there is sufficient weather risk for an outdoor event in September",
    humanRef: "could_change_decision",
    purpose: "Test whether semantic interpretation generalises beyond the market-entry vocabulary.",
  },
  {
    id: "case5-domain-b-supporting",
    decisionTarget: DOMAIN_B_DECISION,
    unknownLabel: "What insurance requirements apply for hosting the event outdoors",
    humanRef: "supports_decision",
    purpose: "Test whether semantic interpretation distinguishes supporting from decisive.",
  },
  {
    id: "case6-domain-b-unrelated",
    decisionTarget: DOMAIN_B_DECISION,
    unknownLabel: "Should the board replace its meeting room chairs next month?",
    humanRef: "unlikely_to_change_decision",
    purpose: "Confirm semantic interpretation does not merely mark everything as relevant.",
  },
];

/* ═══════════════════════════════════════════════════════════
 * Results holder — populated by beforeAll (6 calls total)
 * ═══════════════════════════════════════════════════════════ */

let experimentResults = {};
let inferenceCount = 0;
let timingStats = { min: Infinity, max: 0, total: 0 };
let modelFailureReason = null;

beforeAll(async () => {
  experimentResults = {};

  for (const c of SIX_CASES) {
    const t0 = Date.now();
    try {
      const { result, model } = await semanticInterpret(c.decisionTarget, c.unknownLabel);
      const actualLatency = Date.now() - t0;
      timingStats.min = Math.min(timingStats.min, actualLatency);
      timingStats.max = Math.max(timingStats.max, actualLatency);
      timingStats.total += actualLatency;

      if (!SEMANTIC_CATEGORIES.includes(result.relevance)) {
        result.relevance = "cannot_determine";
      }

      experimentResults[c.id] = {
        decisionTarget: c.decisionTarget,
        unknownLabel: c.unknownLabel,
        humanRef: c.humanRef,
        purpose: c.purpose,
        semanticResult: result.relevance,
        semanticReason: result.reason || "",
        stability: "single_run",
        latencyMs: actualLatency,
        modelUsed: model,
      };
    } catch (e) {
      modelFailureReason = e.message;
      experimentResults[c.id] = {
        decisionTarget: c.decisionTarget,
        unknownLabel: c.unknownLabel,
        humanRef: c.humanRef,
        purpose: c.purpose,
        semanticResult: "cannot_determine",
        semanticReason: `model_failure: ${e.message}`,
        stability: "unstable",
        latencyMs: 0,
        modelUsed: null,
      };
    }
    inferenceCount++;
  }
}, 600000);

/* ═══════════════════════════════════════════════════════════
 * Deterministic baseline — same six cases via existing classifier
 * ═══════════════════════════════════════════════════════════ */

function makeUnknown(id, label) {
  return { id, label, description: label, kind: "unknown", status: "unknown", confidence: "low", value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], childIds: [] };
}

const DETERMINISTIC_RESULTS = {};
beforeAll(() => {
  for (const c of SIX_CASES) {
    const r = assessQuestionRelevanceToDecision({ decisionTarget: c.decisionTarget, unknown: makeUnknown(c.id + "-det", c.unknownLabel) });
    DETERMINISTIC_RESULTS[c.id] = r;
  }
});

/* ═══════════════════════════════════════════════════════════
 * Six cases — semantic interpretation results
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52B — Case 1: familiar relevant question", () => {
  it("semantic interpretation agrees with human reference (could_change_decision)", () => {
    const r = experimentResults["case1-familiar-relevant"];
    expect(r.semanticResult).toBe("could_change_decision");
  });
});

describe("Experiment 52B — Case 2: familiar unrelated question", () => {
  it("semantic interpretation rejects as unlikely_to_change_decision", () => {
    const r = experimentResults["case2-familiar-unrelated"];
    expect(["unlikely_to_change_decision", "cannot_determine"]).toContain(r.semanticResult);
  });
});

describe("Experiment 52B — Case 3: relevant paraphrase (known keyword failure)", () => {
  it("semantic interpretation correctly classifies paraphrase as could_change_decision", () => {
    const r = experimentResults["case3-paraphrase-relevant"];
    expect(r.semanticResult).toBe("could_change_decision");
  });
});

describe("Experiment 52B — Case 4: second-domain relevant question", () => {
  it("semantic interpretation classifies weather risk as could_change_decision", () => {
    const r = experimentResults["case4-domain-b-relevant"];
    expect(r.semanticResult).toBe("could_change_decision");
  });
});

describe("Experiment 52B — Case 5: second-domain supporting question", () => {
  it("semantic interpretation classifies insurance as supports_decision or could_change_decision", () => {
    const r = experimentResults["case5-domain-b-supporting"];
    expect(["supports_decision", "could_change_decision"]).toContain(r.semanticResult);
  });
});

describe("Experiment 52B — Case 6: second-domain unrelated question", () => {
  it("semantic interpretation rejects board chairs as unlikely_to_change_decision", () => {
    const r = experimentResults["case6-domain-b-unrelated"];
    expect(["unlikely_to_change_decision", "cannot_determine"]).toContain(r.semanticResult);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Agreement analysis — semantic vs deterministic vs human
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52B — Agreement: semantic vs deterministic vs human", () => {
  it("reports exact inference count is 6", () => {
    expect(inferenceCount).toBe(6);
  });

  let semanticAgreements = 0;
  let deterministicAgreements = 0;

  for (const c of SIX_CASES) {
    const sem = experimentResults[c.id]?.semanticResult;
    const det = DETERMINISTIC_RESULTS[c.id]?.relevance;
    if (sem === c.humanRef) semanticAgreements++;
    if (det === c.humanRef) deterministicAgreements++;
  }

  it(`semantic interpretation agreed with human reference on ${semanticAgreements}/6 cases`, () => {
    expect(semanticAgreements).toBeGreaterThanOrEqual(0);
    expect(semanticAgreements).toBeLessThanOrEqual(6);
  });

  it(`deterministic baseline agreed with human reference on ${deterministicAgreements}/6 cases`, () => {
    expect(deterministicAgreements).toBeGreaterThanOrEqual(0);
    expect(deterministicAgreements).toBeLessThanOrEqual(6);
  });

  it("records inference timing statistics", () => {
    const successfulCases = SIX_CASES.filter(c => experimentResults[c.id]?.latencyMs > 0).length;
    if (successfulCases === 6) {
      expect(timingStats.min).toBeGreaterThan(0);
      expect(timingStats.max).toBeGreaterThanOrEqual(timingStats.min);
    }
  });
});

/* ═══════════════════════════════════════════════════════════
 * Contract conformance — category validation
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52B — Contract conformance", () => {
  it("all semantic results have a valid category", () => {
    for (const c of SIX_CASES) {
      const r = experimentResults[c.id];
      expect(SEMANTIC_CATEGORIES).toContain(r.semanticResult);
    }
  });
  it("all semantic results include a reason string", () => {
    for (const c of SIX_CASES) {
      const r = experimentResults[c.id];
      expect(typeof r.semanticReason).toBe("string");
    }
  });
});

/* ═══════════════════════════════════════════════════════════
 * Guardrail verification — deterministic classifier unchanged
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52B — Guardrail verification", () => {
  it("deterministic classifier still classifies known market-entry phrasing correctly", () => {
    const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_A_DECISION, unknown: makeUnknown("guard-1", "Whether to enter the European market for analytics tools") });
    expect(r.relevance).toBe("could_change_decision");
  });
  it("deterministic classifier still fails on paraphrase (cannot_determine)", () => {
    const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_A_DECISION, unknown: makeUnknown("guard-2", "Would enough people there actually want what we offer?") });
    expect(r.relevance).toBe("cannot_determine");
  });
  it("semantic instruction unchanged from Experiment 52", () => {
    expect(SEMANTIC_INSTRUCTION.includes("relevance")).toBe(true);
    expect(SEMANTIC_INSTRUCTION.length).toBeGreaterThan(50);
  });
});