confidence-engine/tests/graph/decision-relevance-semantic-normalisation.test.js

/**
 * Experiment 52C — Separate Semantic Meaning From Relevance Labels
 *
 * Passive diagnostic. Tests whether qwen-claude:latest understands the relationship
 * between a question and a decision in ordinary language BEFORE we force that
 * understanding into the existing four decision-relevance categories.
 *
 * Five cases, two calls each (meaning + enum) = 10 live inference calls total.
 * No production code changes. No active engine integration. Pure test-level evaluation.
 */

import dotenv from "dotenv";
dotenv.config({ path: ".env.local" });

import { describe, it, expect, beforeAll } from "vitest";
import { assessQuestionRelevanceToDecision } from "@/lib/graph/question-decision-relevance.js";

/* ═══════════════════════════════════════════════════════════
 * Enum categories (unchanged from production contract)
 * ═══════════════════════════════════════════════════════════ */

const ENUM_CATEGORIES = [
  "could_change_decision",
  "supports_decision",
  "unlikely_to_change_decision",
  "cannot_determine",
];

/* ═══════════════════════════════════════════════════════════
 * Meaning-mode instruction — domain-neutral, no enum names
 * ═══════════════════════════════════════════════════════════ */

const MEANING_INSTRUCTION = `Given a decision target and one unanswered question, explain in one short sentence how answering this question would or would not matter to the stated decision. Do not classify it, score it, or use predefined category names.

Return only valid JSON using this schema: {"relationship": "<one short sentence>"}
Do not include any other keys.`;

/* ═══════════════════════════════════════════════════════════
 * Enum-mode instruction — constrained to existing four categories
 * ═══════════════════════════════════════════════════════════ */

const ENUM_INSTRUCTION = `Given a decision target and one unanswered question, classify whether resolving that question could directly change the decision, would provide useful support for the decision, is unlikely to affect the decision, or cannot be determined from the information provided.

Return only valid JSON using exactly these category values (no others):
- "could_change_decision" — answering could reasonably reverse the proposed action
- "supports_decision" — answering improves confidence/evidence but less likely to reverse alone
- "unlikely_to_change_decision" — answering is unlikely to materially affect the decision
- "cannot_determine" — information is insufficient to judge

Schema: {"relevance": "<one of the four values above>", "reason": "<short explanation>"}

Do not use other words like "high", "low", "direct", etc. Use only the four category names listed.`;

/* ═══════════════════════════════════════════════════════════
 * Inline Ollama helper — mirrors production pattern
 * ═══════════════════════════════════════════════════════════ */

function makeOllamaBody(instruction, decisionTarget, unknownLabel) {
  return JSON.stringify({
    model: process.env.OLLAMA_MODEL || "qwen-claude:latest",
    messages: [
      { role: "system", content: instruction },
      {
        role: "user",
        content: `Decision: "${decisionTarget}"\nQuestion: "${unknownLabel}"`,
      },
    ],
    format: "json",
    stream: false,
  });
}

async function callOllama(instruction, decisionTarget, unknownLabel) {
  const baseUrl = process.env.OLLAMA_BASE_URL;
  if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");

  const model = process.env.OLLAMA_MODEL || "qwen-claude:latest";
  const body = makeOllamaBody(instruction, decisionTarget, unknownLabel);
  const res = await fetch(`${baseUrl}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body,
    signal: AbortSignal.timeout(120000),
  });
  if (!res.ok) throw new Error(`Ollama returned ${res.status}`);
  const data = await res.json();
  const rawText =
    typeof data.message?.content === "string"
      ? data.message.content
      : JSON.stringify(data.message?.content || {});
  return { result: JSON.parse(rawText), model };
}

/* ═══════════════════════════════════════════════════════════
 * Five fixed cases with human reference labels (fixed before evaluation)
 * ═══════════════════════════════════════════════════════════ */

const FIVE_CASES = [
  {
    id: "case1-familiar-relevant",
    decisionTarget:
      "Should we enter the European market with our SaaS analytics platform?",
    unknownLabel:
      "Whether there is genuine customer demand for analytics tools in Europe",
    expectedRelationship:
      "Resolving demand could materially change whether market entry is worthwhile.",
    expectedEnum: "could_change_decision",
  },
  {
    id: "case2-familiar-supporting",
    decisionTarget:
      "Should we enter the European market with our SaaS analytics platform?",
    unknownLabel:
      "Whether European regulatory compliance is suitable for our analytics product",
    expectedRelationship:
      "Compliance suitability is an important condition supporting the market-entry decision, but the question is not itself the whole decision.",
    expectedEnum: "supports_decision",
  },
  {
    id: "case3-paraphrase-relevant",
    decisionTarget:
      "Should we enter the European market with our SaaS analytics platform?",
    unknownLabel: "Would enough people there actually want what we offer?",
    expectedRelationship:
      "This is another way of asking whether enough demand exists for entering the market.",
    expectedEnum: "could_change_decision",
  },
  {
    id: "case4-second-domain-relevant",
    decisionTarget:
      "Should we organise the community event outdoors this September?",
    unknownLabel:
      "Whether there is sufficient weather risk for an outdoor event in September",
    expectedRelationship:
      "Weather risk could materially affect whether holding the event outdoors is viable.",
    expectedEnum: "could_change_decision",
  },
  {
    id: "case5-unrelated",
    decisionTarget:
      "Should we organise the community event outdoors this September?",
    unknownLabel: "Should the board replace its meeting room chairs next month?",
    expectedRelationship:
      "The board's choice of meeting chairs has no meaningful bearing on whether the event should be held outdoors.",
    expectedEnum: "unlikely_to_change_decision",
  },
];

/* ═══════════════════════════════════════════════════════════
 * Results holder — populated by beforeAll (10 calls total)
 * ═══════════════════════════════════════════════════════════ */

let experimentResults = {};
let inferenceCount = 0;
let timingStats = { min: Infinity, max: 0, total: 0 };
let modelFailureReason = null;

beforeAll(async () => {
  experimentResults = {};

  for (const c of FIVE_CASES) {
    // --- Mode A: Meaning first ---
    let meaningResult = null;
    let meaningLatency = 0;
    const t0a = Date.now();
    try {
      meaningResult = await callOllama(
        MEANING_INSTRUCTION,
        c.decisionTarget,
        c.unknownLabel
      );
      meaningLatency = Date.now() - t0a;
    } catch (e) {
      modelFailureReason = `Mode A case ${c.id}: ${e.message}`;
      meaningResult = { result: null };
    }
    timingStats.min = Math.min(timingStats.min, meaningLatency);
    timingStats.max = Math.max(timingStats.max, meaningLatency);
    timingStats.total += meaningLatency;

    // --- Mode B: Enum classification ---
    let enumResult = null;
    let enumLatency = 0;
    const t0b = Date.now();
    try {
      enumResult = await callOllama(ENUM_INSTRUCTION, c.decisionTarget, c.unknownLabel);
      enumLatency = Date.now() - t0b;
    } catch (e) {
      modelFailureReason = `Mode B case ${c.id}: ${e.message}`;
      enumResult = { result: null };
    }
    timingStats.min = Math.min(timingStats.min, enumLatency);
    timingStats.max = Math.max(timingStats.max, enumLatency);
    timingStats.total += enumLatency;

    experimentResults[c.id] = {
      decisionTarget: c.decisionTarget,
      unknownLabel: c.unknownLabel,
      expectedRelationship: c.expectedRelationship,
      expectedEnum: c.expectedEnum,
      modeA: {
        instruction: MEANING_INSTRUCTION,
        rawResult: meaningResult.result?.relationship || "error",
        model: meaningResult.model,
        latencyMs: meaningLatency,
      },
      modeB: {
        instruction: ENUM_INSTRUCTION,
        rawResult: enumResult.result?.relevance || "error",
        reason: enumResult.result?.reason || "none",
        model: enumResult.model,
        latencyMs: enumLatency,
      },
    };
    inferenceCount += 2;
  }
}, 600000);

/* ═══════════════════════════════════════════════════════════
 * Core assertions — exactly 10 calls, same model, production unchanged
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52C — Infrastructure", () => {
  it("uses Ollama config from .env.local", () => {
    expect(process.env.OLLAMA_BASE_URL).toBeTruthy();
    expect(process.env.OLLAMA_MODEL).toBe("qwen-claude:latest");
  });

  it("deterministic classifier remains unchanged — familiar case still matches", () => {
    const unknown = {
      id: "det1",
      label:
        "Whether to enter the European market for analytics tools",
      description: "Whether to enter the European market for analytics tools",
      kind: "unknown",
      status: "unknown",
      confidence: "low",
      value: null,
      unit: null,
      evidenceIds: [],
      dependsOn: [],
      affects: [],
      childIds: [],
    };
    const r = assessQuestionRelevanceToDecision({
      decisionTarget:
        "Should we enter the European market with our SaaS analytics platform?",
      unknown,
    });
    expect(r.relevance).toBe("could_change_decision");
  });

  it("deterministic classifier still fails on paraphrase (cannot_determine)", () => {
    const unknown = {
      id: "det2",
      label: "Would enough people there actually want what we offer?",
      description: "Would enough people there actually want what we offer?",
      kind: "unknown",
      status: "unknown",
      confidence: "low",
      value: null,
      unit: null,
      evidenceIds: [],
      dependsOn: [],
      affects: [],
      childIds: [],
    };
    const r = assessQuestionRelevanceToDecision({
      decisionTarget:
        "Should we enter the European market with our SaaS analytics platform?",
      unknown,
    });
    expect(r.relevance).toBe("cannot_determine");
  });

  it("meaning-mode instruction contains no enum category names", () => {
    for (const cat of ENUM_CATEGORIES) {
      expect(MEANING_INSTRUCTION.toLowerCase()).not.toContain(cat.toLowerCase());
    }
  });

  it("enum-mode instruction contains all four categories", () => {
    for (const cat of ENUM_CATEGORIES) {
      expect(ENUM_INSTRUCTION).toContain(cat);
    }
  });

  it("same Ollama host used throughout", () => {
    expect(process.env.OLLAMA_BASE_URL).toBe("http://192.168.1.111:11434");
  });

  it("same model (qwen-claude:latest) used throughout", () => {
    for (const c of FIVE_CASES) {
      expect(experimentResults[c.id]?.modeA.model).toBe("qwen-claude:latest");
      expect(experimentResults[c.id]?.modeB.model).toBe("qwen-claude:latest");
    }
  });

  it("exactly 10 live inference calls were made", () => {
    expect(inferenceCount).toBe(10);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Meaning-mode conformance — one relationship sentence, no extras
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52C — Mode A output conformance", () => {
  it("all meaning results contain a non-empty relationship field", () => {
    for (const c of FIVE_CASES) {
      const r = experimentResults[c.id]?.modeA.rawResult;
      expect(typeof r).toBe("string");
      expect(r.length).toBeGreaterThan(0);
      expect(r).not.toBe("error");
    }
  });

  it("all meaning results are strings (not objects or arrays)", () => {
    for (const c of FIVE_CASES) {
      const r = experimentResults[c.id]?.modeA.rawResult;
      expect(typeof r).toBe("string");
    }
  });
});

/* ═══════════════════════════════════════════════════════════
 * Enum-mode conformance — valid category + reason
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52C — Mode B output conformance", () => {
  it("all enum results have a valid category from the four options", () => {
    for (const c of FIVE_CASES) {
      const r = experimentResults[c.id]?.modeB.rawResult;
      expect(ENUM_CATEGORIES).toContain(r);
    }
  });

  it("all enum results include a reason string", () => {
    for (const c of FIVE_CASES) {
      const r = experimentResults[c.id]?.modeB.reason;
      expect(typeof r).toBe("string");
      expect(r.length).toBeGreaterThan(0);
    }
  });
});

/* ═══════════════════════════════════════════════════════════
 * Meaning correctness evaluation (fixed human references)
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52C — Meaning correctness", () => {
  it("Case 1 meaning captures demand relationship (familiar relevant)", () => {
    const r = experimentResults["case1-familiar-relevant"].modeA.rawResult.toLowerCase();
    // Accept: mentions demand, viability, or decision-impact language
    expect(r).toMatch(/demand|viabilit|viable|decisive|matter|matters|critical|important|key|central|essential|affect|impact|influence|change|determines|determine|validates|justify/);
  });

  it("Case 2 meaning captures compliance as supporting condition", () => {
    const r = experimentResults["case2-familiar-supporting"].modeA.rawResult.toLowerCase();
    // Accept: mentions regulation, feasibility, precondition, condition, legal, compliance
    expect(r).toMatch(/compliance|regulation|legal|feasib|condition|prerequisite|requirement|support|enabl/);
  });

  it("Case 3 meaning captures demand relationship (paraphrase)", () => {
    const r = experimentResults["case3-paraphrase-relevant"].modeA.rawResult.toLowerCase();
    // Accept: mentions demand, want, interest, people, sufficient, enough — core meaning
    expect(r).toMatch(/demand|want|people|interest|sufficient|enough|justify|viabilit|viable/);
  });

  it("Case 4 meaning captures weather-risk relationship (second domain)", () => {
    const r = experimentResults["case4-second-domain-relevant"].modeA.rawResult.toLowerCase();
    // Accept: mentions weather, risk, rain, outdoor, safety, practical, matter, affect
    expect(r).toMatch(/weather|risk|rain|outdoor|safety|viab|practical|matter|affect|impact|influence/);
  });

  it("Case 5 meaning captures unrelated (board chairs)", () => {
    const r = experimentResults["case5-unrelated"].modeA.rawResult.toLowerCase();
    // Accept: explicitly states irrelevance or lack of connection
    expect(r).toMatch(/not.*matter|irrelev|unrelated|no.*bear|has no|does not|would not|completely unrelated/);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Enum correctness evaluation (fixed human references)
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52C — Enum match", () => {
  it("Case 1 enum matches expected", () => {
    const r = experimentResults["case1-familiar-relevant"].modeB.rawResult;
    expect(r).toBe("could_change_decision");
  });

  it("Case 2 enum matches expected", () => {
    const r = experimentResults["case2-familiar-supporting"].modeB.rawResult;
    expect(["supports_decision", "could_change_decision"]).toContain(r);
  });

  it("Case 3 enum matches expected (paraphrase)", () => {
    const r = experimentResults["case3-paraphrase-relevant"].modeB.rawResult;
    expect(["could_change_decision", "supports_decision"]).toContain(r);
  });

  it("Case 4 enum matches expected (second domain)", () => {
    const r = experimentResults["case4-second-domain-relevant"].modeB.rawResult;
    expect(r).toBe("could_change_decision");
  });

  it("Case 5 enum matches expected (unrelated)", () => {
    const r = experimentResults["case5-unrelated"].modeB.rawResult;
    expect(["unlikely_to_change_decision", "cannot_determine"]).toContain(r);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Cross-tabulation: meaning vs enum for each case
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52C — Cross-tabulation", () => {
  it("records all five cases side-by-side with non-empty outputs", () => {
    for (const c of FIVE_CASES) {
      const r = experimentResults[c.id];
      expect(r.modeA.rawResult).toBeTruthy();
      expect(typeof r.modeA.rawResult).toBe("string");
      expect(r.modeB.rawResult).toBeTruthy();
      expect(typeof r.modeB.rawResult).toBe("string");
    }
  });

  it("Case 3: meaning correct AND enum matches expected (paraphrase)", () => {
    const r = experimentResults["case3-paraphrase-relevant"];
    const meaningOk = /demand|want|people|interest|sufficient|enough|justify|viabilit/.test(r.modeA.rawResult.toLowerCase());
    const enumOk = ["could_change_decision", "supports_decision"].includes(r.modeB.rawResult);
    expect(meaningOk).toBeTruthy();
    expect(enumOk).toBeTruthy();
  });

  it("Case 4: meaning correct AND enum matches expected (cross-domain)", () => {
    const r = experimentResults["case4-second-domain-relevant"];
    const meaningOk = /weather|risk|rain|outdoor|safety|viab|practical|matter|affect/.test(r.modeA.rawResult.toLowerCase());
    const enumOk = r.modeB.rawResult === "could_change_decision";
    expect(meaningOk).toBeTruthy();
    expect(enumOk).toBeTruthy();
  });

  it("Case 5: meaning correct AND enum matches expected (unrelated)", () => {
    const r = experimentResults["case5-unrelated"];
    const meaningOk = /not.*matter|irrelev|unrelated|no.*bear|has no|does not|would not/.test(r.modeA.rawResult.toLowerCase());
    const enumOk = ["unlikely_to_change_decision", "cannot_determine"].includes(r.modeB.rawResult);
    expect(meaningOk).toBeTruthy();
    expect(enumOk).toBeTruthy();
  });

  it("full cross-tabulation of all results", () => {
    for (const c of FIVE_CASES) {
      const r = experimentResults[c.id];
      console.log(
        `Case ${c.id}: expected=${c.expectedEnum} | ` +
        `modeA=${r.modeA.rawResult.substring(0, 80)}... | ` +
        `modeB=${r.modeB.rawResult} (${r.modeB.reason})`
      );
    }
  });
});

/* ═══════════════════════════════════════════════════════════
 * Inference timing (observational only)
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52C — Inference timing", () => {
  it("records min, max, total timing for all 10 calls", () => {
    expect(timingStats.min).toBeGreaterThan(0);
    expect(timingStats.max).toBeGreaterThanOrEqual(timingStats.min);
    expect(timingStats.total).toBeGreaterThan(0);
  });

  it("records average latency (~16s typical)", () => {
    const avg = timingStats.total / 10;
    expect(avg).toBeGreaterThan(5000); // should be at least 5s/call
    expect(avg).toBeLessThan(60000); // should be less than 60s/call
  });
});