confidence-engine/tests/reconstruction/semantic-hypothesis-evidence-needs.test.js

import { describe, it, expect } from "vitest";
import { config } from "dotenv";
import path from "path";
import { fileURLToPath } from "url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
config({ path: path.resolve(__dirname, "../../.env.local") });

const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL;
const OLLAMA_MODEL = process.env.OLLAMA_MODEL;

if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) {
  throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local");
}

/**
 * Make one live Ollama chat call for evidence-need comparison.
 */
async function callEvidenceModel(problem, hypothesisA, hypothesisB) {
  const instruction = `Compare the evidence needed to investigate the two hypotheses. Return sameEvidenceNeeded: true only when materially the same evidence would test both hypotheses. Return false when each hypothesis requires meaningfully different evidence, even if both are trying to explain the same overall problem. List the main evidence needed for each hypothesis. Do not decide which hypothesis is correct and do not generate questions.

Return valid JSON only in this shape:
{
  "sameEvidenceNeeded": true | false,
  "evidenceForA": ["..."],
  "evidenceForB": ["..."]
}`;

  const messages = [
    { role: "system", content: instruction.trim() },
    {
      role: "user",
      content: `Problem: ${JSON.stringify(problem)}

Hypothesis A: ${JSON.stringify(hypothesisA)}

Hypothesis B: ${JSON.stringify(hypothesisB)}`,
    },
  ];

  const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model: OLLAMA_MODEL,
      messages,
      format: "json",
      stream: false,
    }),
  });

  if (!res.ok) {
    throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
  }

  const data = await res.json();
  const rawContent = data.message?.content ?? "";
  const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");

  return JSON.parse(cleaned.trim());
}

// ──────────────────────────────────────────────
// Fixed human-reference ground truth (pre-written)
// ──────────────────────────────────────────────

const CASES = [
  {
    id: "Case 1 — Delivery Delay / Different Evidence",
    problem: "Orders are arriving late and customers have started complaining.",
    hypothesisA: "Delivery delays are being caused by insufficient staff capacity.",
    hypothesisB: "Delivery delays are being caused by unreliable supplier lead times.",
    reference: { sameEvidenceNeeded: false },
  },
  {
    id: "Case 2 — Same Cause, Paraphrased / Same Evidence",
    problem: "Orders are arriving late and customers have started complaining.",
    hypothesisA: "The team may not have enough capacity to process orders on time.",
    hypothesisB: "Insufficient staff capacity may be causing the order delays.",
    reference: { sameEvidenceNeeded: true },
  },
  {
    id: "Case 3 — Different Causes, Different Domain",
    problem: "Website sales have fallen sharply over the last month.",
    hypothesisA: "The fall may be caused by a recent increase in product prices.",
    hypothesisB: "The fall may be caused by a technical checkout problem.",
    reference: { sameEvidenceNeeded: false },
  },
];

// ──────────────────────────────────────────────
// Semantic evaluation against fixed human references
// ──────────────────────────────────────────────

function evaluateEvidenceNeed(modelResult, reference) {
  const result = modelResult;

  let issues = [];
  let notes = [];

  // Boolean must match reference
  if (result.sameEvidenceNeeded !== reference.sameEvidenceNeeded) {
    issues.push("boolean_mismatch: model evidence-need assessment does not match fixed human reference");
  }

  // Must provide evidence lists
  if (!Array.isArray(result.evidenceForA) || result.evidenceForA.length === 0) {
    issues.push("missing_evidence_A: evidenceForA is empty or not an array");
  }
  if (!Array.isArray(result.evidenceForB) || result.evidenceForB.length === 0) {
    issues.push("missing_evidence_B: evidenceForB is empty or not an array");
  }

  // Check for invariant violations (no winner, no next question)
  const allText = JSON.stringify(result).toLowerCase();
  const hasWinnerSelection = allText.includes("preferred") ||
    allText.includes("more likely") ||
    allText.includes("should go with");
  if (hasWinnerSelection) {
    issues.push("invariant_failed: model appears to have chosen a winning hypothesis");
  }

  const hasNextQuestion = allText.includes("ask the user") ||
    allText.includes("ask next") ||
    allText.includes("question to ask");
  if (hasNextQuestion) {
    notes.push("caution: model generated a next-question suggestion alongside the evidence assessment");
  }

  // Semantic check: if sameEvidenceNeeded=false, evidenceForA and evidenceForB should be materially different
  if (reference.sameEvidenceNeeded === false && Array.isArray(result.evidenceForA) && Array.isArray(result.evidenceForB)) {
    const aText = result.evidenceForA.join(" ").toLowerCase();
    const bText = result.evidenceForB.join(" ").toLowerCase();
    if (aText === bText) {
      notes.push("caution: evidenceForA and evidenceForB are identical despite sameEvidenceNeeded=false");
    }
  }

  if (issues.length === 0) return "evidence_need_correct";
  return "evidence_need_failed";
}

// ──────────────────────────────────────────────
// Describe the experiment as a single test suite
// ──────────────────────────────────────────────

describe("Experiment 54O — Evidence Needs Across Competing Hypotheses (test-only)", () => {
  const results = [];
  const timings = [];

  for (const testCase of CASES) {
    it(`${testCase.id} — evidence need comparison`, async () => {
      const t0 = performance.now();
      const result = await callEvidenceModel(
        testCase.problem,
        testCase.hypothesisA,
        testCase.hypothesisB
      );
      const elapsed = performance.now() - t0;
      timings.push(elapsed);

      expect(result).toHaveProperty("sameEvidenceNeeded");
      expect(typeof result.sameEvidenceNeeded).toBe("boolean");
      expect(Array.isArray(result.evidenceForA)).toBe(true);
      expect(Array.isArray(result.evidenceForB)).toBe(true);

      const classification = evaluateEvidenceNeed(result, testCase.reference);

      results.push({
        id: testCase.id,
        problem: testCase.problem,
        hypothesisA: testCase.hypothesisA,
        hypothesisB: testCase.hypothesisB,
        reference: testCase.reference,
        modelResult: result,
        classification: classification,
        timingMs: Number(elapsed.toFixed(2)),
      });

      console.log(`\n=== ${testCase.id} ===`);
      console.log(`Model output:`);
      console.log(`  sameEvidenceNeeded:`, result.sameEvidenceNeeded);
      console.log(`  evidenceForA:`, result.evidenceForA);
      console.log(`  evidenceForB:`, result.evidenceForB);
      console.log(`Reference:`, testCase.reference);
      console.log(`Classification: ${classification}`);
    }, 120000);
  }

  it("54O — summary and required questions", () => {
    const correct = results.filter((r) => r.classification === "evidence_need_correct").length;
    const failed = results.filter((r) => r.classification === "evidence_need_failed").length;
    const total = timings.reduce((a, b) => a + b, 0);
    const avg = total / timings.length;
    const fastest = Math.min(...timings);
    const slowest = Math.max(...timings);

    console.log("\n=== Experiment 54O Summary ===");
    console.log(`Cases: ${results.length}`);
    console.log(`Evidence-need-correct: ${correct}, Evidence-need-failed: ${failed}`);

    results.forEach((r) => {
      const expectedLabel = r.reference.sameEvidenceNeeded ? "same" : "different";
      const resultLabel = r.modelResult.sameEvidenceNeeded ? "same" : "different";
      const match = r.classification === "evidence_need_correct" ? "✓" : "✗";
      console.log(`  ${match} ${r.id}: expected ${expectedLabel}, model said ${resultLabel}`);
    });

    // Check for invariant violations across all results
    let winnerChosen = false;
    let questionGenerated = false;
    for (const r of results) {
      const text = JSON.stringify(r.modelResult).toLowerCase();
      if (text.includes("preferred") || text.includes("more likely")) {
        winnerChosen = true;
      }
      if (text.includes("ask the user") || text.includes("question to ask")) {
        questionGenerated = true;
      }
    }

    console.log(`Total inference time: ${total.toFixed(2)}ms`);
    console.log(`Average: ${avg.toFixed(2)}ms, Fastest: ${fastest.toFixed(2)}ms, Slowest: ${slowest.toFixed(2)}ms`);
    console.log(`Winner chosen by model: ${winnerChosen ? "yes" : "no"}`);
    console.log(`Next question generated: ${questionGenerated ? "yes" : "no"}`);

    expect(false).toBe(false); // Q1-Q10 addressed in report
    expect(results.length).toBe(3);
  });
}, 600000);