confidence-engine/tests/reconstruction/semantic-disagreement-consequence.test.js

import { describe, it, expect } from "vitest";
import { config } from "dotenv";
import path from "path";
import { fileURLToPath } from "url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
config({ path: path.resolve(__dirname, "../../.env.local") });

const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL;
const OLLAMA_MODEL = process.env.OLLAMA_MODEL;

if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) {
  throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local");
}

/**
 * Make one live Ollama chat call for consequence detection.
 */
async function callConsequenceModel(source, sharedMeaning, disagreement) {
  const instruction = `Decide whether the stated disagreement would materially change what information needs to be established next before reasoning can proceed confidently. Return true only when the competing interpretations imply meaningfully different evidence or investigation directions. Return false when the disagreement is only wording, emphasis, or does not change the information needed next. Do not choose which interpretation is correct and do not generate a next question.

Return valid JSON only in this shape:
{
  "changesInformationNeededNext": true | false,
  "reason": "one short sentence explaining why"
}`;

  const messages = [
    { role: "system", content: instruction.trim() },
    {
      role: "user",
      content: `Source: "${source}"

Shared meaning both interpretations carry: ${JSON.stringify(sharedMeaning)}

Disagreement between the two interpretations: ${JSON.stringify(disagreement)}`,
    },
  ];

  const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model: OLLAMA_MODEL,
      messages,
      format: "json",
      stream: false,
    }),
  });

  if (!res.ok) {
    throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
  }

  const data = await res.json();
  const rawContent = data.message?.content ?? "";
  const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");

  return JSON.parse(cleaned.trim());
}

// ──────────────────────────────────────────────
// Fixed human-reference ground truth (pre-written)
// ──────────────────────────────────────────────

const CASES = [
  {
    id: "Case 1 — Pricing Strength Versus Unresolved Cause",
    source: "Revenue is down. I think pricing may be part of the problem, but I am not sure.",
    sharedMeaning: ["revenue has declined", "pricing may be related to the problem"],
    disagreement: [
      "one interpretation treats pricing as a potentially material contributor",
      "the other keeps pricing unresolved and allows other causes"
    ],
    reference: {
      changesInformationNeededNext: true,
      reason: "If pricing is materially causal, pricing evidence becomes central. If causality remains broad, other possible causes also need investigation."
    }
  },
  {
    id: "Case 2 — Paraphrase / No Material Disagreement",
    source: "Revenue is down. I think pricing may be part of the problem, but I am not sure.",
    sharedMeaning: ["revenue has declined", "pricing may contribute", "its importance remains uncertain"],
    disagreement: ["no substantive disagreement; wording differs only"],
    reference: {
      changesInformationNeededNext: false,
      reason: "Equivalent interpretations should not cause a different investigation merely because they are phrased differently."
    }
  },
  {
    id: "Case 3 — Competing Causes",
    source: "Orders are arriving late and customers have started complaining.",
    sharedMeaning: ["orders are arriving late", "there is a delivery-delay problem"],
    disagreement: [
      "one interpretation attributes the likely cause to insufficient staff capacity",
      "the other attributes the possible cause to unreliable supplier lead times"
    ],
    reference: {
      changesInformationNeededNext: true,
      reason: "Staff-capacity evidence and supplier-lead-time evidence are materially different investigation directions."
    }
  }
];

// ──────────────────────────────────────────────
// Semantic evaluation against fixed human references
// ──────────────────────────────────────────────

function evaluateConsequence(modelResult, reference) {
  const result = modelResult;

  let issues = [];
  let notes = [];

  // Boolean must match reference
  if (result.changesInformationNeededNext !== reference.changesInformationNeededNext) {
    issues.push("boolean_mismatch: model consequence does not match fixed human reference");
  }

  // Reason must explain information-need divergence or non-divergence
  const reasonStr = (result.reason || "").toLowerCase();
  if (!reasonStr || reasonStr.length < 5) {
    issues.push("missing_reason: reason is empty or too short");
  }

  // Check for invariant violations (no winner, no scores, no questions)
  const hasWinnerSelection = reasonStr.includes("interpretation a") && reasonStr.includes("correct") ||
                             reasonStr.includes("interpretation b") && reasonStr.includes("correct") ||
                             reasonStr.includes("winner");
  if (hasWinnerSelection) {
    issues.push("invariant_failed: model appears to have chosen a winning interpretation");
  }

  const hasNextQuestion = reasonStr.includes("ask") || reasonStr.includes("question") ||
                          reasonStr.includes("next step") || reasonStr.includes("follow up");
  if (hasNextQuestion) {
    notes.push("caution: model generated a next-question suggestion alongside the consequence judgment");
  }

  const hasScores = result.score !== undefined || result.confidence !== undefined;
  if (hasScores) {
    issues.push("invariant_failed: output contains score or confidence fields");
  }

  if (issues.length === 0) return "consequence_correct";
  return "consequence_failed";
}

// ──────────────────────────────────────────────
// Describe the experiment as a single test suite
// ──────────────────────────────────────────────

describe("Experiment 54N — Consequence of Interpretation Disagreement (test-only)", () => {
  const results = [];
  const timings = [];

  for (const testCase of CASES) {
    it(`${testCase.id} — consequence detection`, async () => {
      const t0 = performance.now();
      const result = await callConsequenceModel(
        testCase.source,
        testCase.sharedMeaning,
        testCase.disagreement
      );
      const elapsed = performance.now() - t0;
      timings.push(elapsed);

      expect(result).toHaveProperty("changesInformationNeededNext");
      expect(typeof result.changesInformationNeededNext).toBe("boolean");
      expect(result).toHaveProperty("reason");
      expect(typeof result.reason).toBe("string");

      const classification = evaluateConsequence(result, testCase.reference);

      results.push({
        id: testCase.id,
        source: testCase.source,
        sharedMeaning: testCase.sharedMeaning,
        disagreement: testCase.disagreement,
        reference: testCase.reference,
        modelResult: result,
        classification: classification,
        timingMs: Number(elapsed.toFixed(2)),
      });

      console.log(`\n=== ${testCase.id} ===`);
      console.log(`Model output:`);
      console.log(`  changesInformationNeededNext:`, result.changesInformationNeededNext);
      console.log(`  reason: "${result.reason}"`);
      console.log(`Reference:`, testCase.reference);
      console.log(`Classification: ${classification}`);
    }, 120000);
  }

  it("54N — summary and required questions", () => {
    const correct = results.filter((r) => r.classification === "consequence_correct").length;
    const failed = results.filter((r) => r.classification === "consequence_failed").length;
    const total = timings.reduce((a, b) => a + b, 0);
    const avg = total / timings.length;
    const fastest = Math.min(...timings);
    const slowest = Math.max(...timings);

    console.log("\n=== Experiment 54N Summary ===");
    console.log(`Cases: ${results.length}`);
    console.log(`Consequence-correct: ${correct}, Consequence-failed: ${failed}`);

    results.forEach((r) => {
      const label = r.reference.changesInformationNeededNext ? "should change" : "should NOT change";
      const resultStr = r.modelResult.changesInformationNeededNext ? "changed" : "did not change";
      const match = r.classification === "consequence_correct" ? "✓" : "✗";
      console.log(`  ${match} ${r.id}: expected ${label}, model said ${resultStr}`);
    });

    // Check for invariant violations across all results
    let winnerChosen = false;
    let questionGenerated = false;
    for (const r of results) {
      const reasonStr = r.modelResult.reason?.toLowerCase() || "";
      if (reasonStr.includes("winner") || (reasonStr.includes("interpretation a") && reasonStr.includes("correct"))) {
        winnerChosen = true;
      }
      if (reasonStr.includes("ask") || reasonStr.includes("question")) {
        questionGenerated = true;
      }
    }

    console.log(`Total inference time: ${total.toFixed(2)}ms`);
    console.log(`Average: ${avg.toFixed(2)}ms, Fastest: ${fastest.toFixed(2)}ms, Slowest: ${slowest.toFixed(2)}ms`);
    console.log(`Winner chosen by model: ${winnerChosen ? "yes" : "no"}`);
    console.log(`Next question generated: ${questionGenerated ? "yes" : "no"}`);

    // Required question answers (all pass — these are questions to be answered in the report)
    expect(false).toBe(false); // Q1-Q7 addressed in report
    expect(false).toBe(false); // Q8-Q10 must be answered No

    // Final assertion — always pass so timing/totals are recorded
    expect(results.length).toBe(3);
  });
}, 600000);