confidence-engine/tests/reconstruction/semantic-evidence-consequence.test.js

import { describe, it, expect } from "vitest";
import { config } from "dotenv";
import path from "path";
import { fileURLToPath } from "url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
config({ path: path.resolve(__dirname, "../../.env.local") });

const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL;
const OLLAMA_MODEL = process.env.OLLAMA_MODEL;

if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) {
  throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local");
}

/**
 * Make one live Ollama chat call for evidence-consequence detection.
 */
async function callConsequenceModel(problem, evidenceForA, evidenceForB) {
  const instruction = `Decide whether these two evidence sets imply materially different information must be established before reasoning can proceed confidently. Return true when the evidence sets investigate meaningfully different things. Return false when they are materially the same despite wording differences. Do not choose which hypothesis is correct and do not generate a question.

Return valid JSON only in this shape:
{
  "changesInformationNeededNext": true | false,
  "reason": "one short sentence explaining why"
}`;

  const messages = [
    { role: "system", content: instruction.trim() },
    {
      role: "user",
      content: `Problem: ${JSON.stringify(problem)}

Evidence for hypothesis A: ${JSON.stringify(evidenceForA)}

Evidence for hypothesis B: ${JSON.stringify(evidenceForB)}`,
    },
  ];

  const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model: OLLAMA_MODEL,
      messages,
      format: "json",
      stream: false,
    }),
  });

  if (!res.ok) {
    throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
  }

  const data = await res.json();
  const rawContent = data.message?.content ?? "";
  const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");

  return JSON.parse(cleaned.trim());
}

// ──────────────────────────────────────────────
// Fixed human-reference ground truth (pre-written)
// ──────────────────────────────────────────────

const CASES = [
  {
    id: "Case 1 — Delivery Causes / Different Evidence (54N failure case, explicit evidence)",
    problem: "Orders are arriving late and customers have started complaining.",
    evidenceForA: [
      "staffing levels",
      "shift coverage",
      "workload",
      "order-processing throughput",
    ],
    evidenceForB: [
      "supplier delivery records",
      "supplier lead times",
      "supplier reliability history",
    ],
    reference: {
      changesInformationNeededNext: true,
      reason: "Staffing/capacity evidence and supplier lead-time/reliability evidence investigate materially different causes of delay.",
    },
  },
  {
    id: "Case 2 — Same Staffing Cause / Paraphrased Evidence",
    problem: "Orders are arriving late and customers have started complaining.",
    evidenceForA: [
      "staffing levels",
      "shift coverage",
      "workload",
      "processing times",
    ],
    evidenceForB: [
      "team capacity",
      "staffing levels",
      "shift coverage",
      "order-processing times",
    ],
    reference: {
      changesInformationNeededNext: false,
      reason: "Both evidence sets investigate the same underlying staffing/capacity question despite paraphrased terminology.",
    },
  },
  {
    id: "Case 3 — Pricing Versus Checkout / Different Evidence",
    problem: "Website sales have fallen sharply over the last month.",
    evidenceForA: [
      "price changes",
      "conversion response after price changes",
      "customer price sensitivity",
      "competitor pricing",
    ],
    evidenceForB: [
      "checkout error logs",
      "checkout funnel drop-off",
      "payment failures",
      "browser/device failures",
    ],
    reference: {
      changesInformationNeededNext: true,
      reason: "Pricing/conversion evidence and checkout/technical failure evidence investigate materially different causes of sales decline.",
    },
  },
];

// ──────────────────────────────────────────────
// Semantic evaluation against fixed human references
// ──────────────────────────────────────────────

function evaluateConsequence(modelResult, reference) {
  const result = modelResult;

  let issues = [];
  let notes = [];

  // Boolean must match reference
  if (result.changesInformationNeededNext !== reference.changesInformationNeededNext) {
    issues.push("boolean_mismatch: model consequence does not match fixed human reference");
  }

  // Reason must explain information/evidence distinction
  const reasonStr = (result.reason || "").toLowerCase();
  if (!reasonStr || reasonStr.length < 5) {
    issues.push("missing_reason: reason is empty or too short");
  } else {
    // Manual semantic review: check that reason addresses evidence/information distinction
    const discussesEvidence = reasonStr.includes("evidence") ||
      reasonStr.includes("information") ||
      reasonStr.includes("investigat") ||
      reasonStr.includes("differ") ||
      reasonStr.includes("same") ||
      reasonStr.includes("cause") ||
      reasonStr.includes("understand");
    if (!discussesEvidence) {
      notes.push("caution: reason does not clearly address the evidence/information distinction");
    }
  }

  // Check for invariant violations (no winner, no next question, no scores)
  const hasWinnerSelection = reasonStr.includes("interpretation a") && reasonStr.includes("correct") ||
                             reasonStr.includes("interpretation b") && reasonStr.includes("correct") ||
                             reasonStr.includes("winner");
  if (hasWinnerSelection) {
    issues.push("invariant_failed: model appears to have chosen a winning hypothesis");
  }

  const hasNextQuestion = reasonStr.includes("ask") || reasonStr.includes("question") ||
                          reasonStr.includes("next step") || reasonStr.includes("follow up");
  if (hasNextQuestion) {
    notes.push("caution: model generated a next-question suggestion alongside the consequence judgment");
  }

  const hasScores = result.score !== undefined || result.confidence !== undefined;
  if (hasScores) {
    issues.push("invariant_failed: output contains score or confidence fields");
  }

  if (issues.length === 0) return "consequence_correct";
  return "consequence_failed";
}

// ──────────────────────────────────────────────
// Describe the experiment as a single test suite
// ──────────────────────────────────────────────

describe("Experiment 54P — Evidence-to-Consequence from Explicit Evidence Needs (test-only)", () => {
  const results = [];
  const timings = [];

  for (const testCase of CASES) {
    it(`${testCase.id} — consequence from explicit evidence`, async () => {
      const t0 = performance.now();
      const result = await callConsequenceModel(
        testCase.problem,
        testCase.evidenceForA,
        testCase.evidenceForB
      );
      const elapsed = performance.now() - t0;
      timings.push(elapsed);

      expect(result).toHaveProperty("changesInformationNeededNext");
      expect(typeof result.changesInformationNeededNext).toBe("boolean");
      expect(result).toHaveProperty("reason");
      expect(typeof result.reason).toBe("string");

      const classification = evaluateConsequence(result, testCase.reference);

      results.push({
        id: testCase.id,
        problem: testCase.problem,
        evidenceForA: testCase.evidenceForA,
        evidenceForB: testCase.evidenceForB,
        reference: testCase.reference,
        modelResult: result,
        classification: classification,
        timingMs: Number(elapsed.toFixed(2)),
      });

      console.log(`\n=== ${testCase.id} ===`);
      console.log(`Model output:`);
      console.log(`  changesInformationNeededNext:`, result.changesInformationNeededNext);
      console.log(`  reason: "${result.reason}"`);
      console.log(`Reference:`, testCase.reference);
      console.log(`Classification: ${classification}`);
    }, 120000);
  }

  it("54P — summary and required questions", () => {
    const correct = results.filter((r) => r.classification === "consequence_correct").length;
    const failed = results.filter((r) => r.classification === "consequence_failed").length;
    const total = timings.reduce((a, b) => a + b, 0);
    const avg = total / timings.length;
    const fastest = Math.min(...timings);
    const slowest = Math.max(...timings);

    console.log("\n=== Experiment 54P Summary ===");
    console.log(`Cases: ${results.length}`);
    console.log(`Consequence-correct: ${correct}, Consequence-failed: ${failed}`);

    results.forEach((r) => {
      const expectedLabel = r.reference.changesInformationNeededNext ? "should change" : "should NOT change";
      const resultStr = r.modelResult.changesInformationNeededNext ? "changed" : "did not change";
      const match = r.classification === "consequence_correct" ? "✓" : "✗";
      console.log(`  ${match} ${r.id}: expected ${expectedLabel}, model said ${resultStr}`);
    });

    // Check for invariant violations across all results
    let winnerChosen = false;
    let questionGenerated = false;
    for (const r of results) {
      const reasonStr = r.modelResult.reason?.toLowerCase() || "";
      if (reasonStr.includes("winner") || (reasonStr.includes("interpretation a") && reasonStr.includes("correct"))) {
        winnerChosen = true;
      }
      if (reasonStr.includes("ask") || reasonStr.includes("question")) {
        questionGenerated = true;
      }
    }

    console.log(`Total inference time: ${total.toFixed(2)}ms`);
    console.log(`Average: ${avg.toFixed(2)}ms, Fastest: ${fastest.toFixed(2)}ms, Slowest: ${slowest.toFixed(2)}ms`);
    console.log(`Winner chosen by model: ${winnerChosen ? "yes" : "no"}`);
    console.log(`Next question generated: ${questionGenerated ? "yes" : "no"}`);

    // Required question answers (all pass — these are questions to be answered in the report)
    expect(false).toBe(false); // Q1-Q9 addressed in report
    expect(results.length).toBe(3);
  });
}, 600000);