confidence-engine/tests/reconstruction/semantic-structured-evidence-consequence.test.js

import { describe, it, expect } from "vitest";
import { config } from "dotenv";
import path from "path";
import { fileURLToPath } from "url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
config({ path: path.resolve(__dirname, "../../.env.local") });

const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL;
const OLLAMA_MODEL = process.env.OLLAMA_MODEL;

if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) {
  throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local");
}

/**
 * Make one live Ollama chat call: identify evidence needed for each hypothesis,
 * then decide whether those evidence needs mean materially different information
 * must be established next.
 */
async function callStructuredEvidenceConsequence(problem, hypothesisA, hypothesisB) {
  const instruction = `Identify the main evidence needed to investigate each hypothesis. Then decide whether those evidence needs mean materially different information must be established next. Return true when the evidence sets investigate meaningfully different things and false when they are materially the same despite wording differences. Do not choose which hypothesis is correct and do not generate a question.

Return valid JSON only in this shape:
{
  "evidenceForA": ["short evidence needs"],
  "evidenceForB": ["short evidence needs"],
  "changesInformationNeededNext": true | false,
  "reason": "one short sentence"
}`;

  const messages = [
    { role: "system", content: instruction.trim() },
    {
      role: "user",
      content: `Problem: ${JSON.stringify(problem)}

Hypothesis A: ${JSON.stringify(hypothesisA)}

Hypothesis B: ${JSON.stringify(hypothesisB)}`,
    },
  ];

  const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model: OLLAMA_MODEL,
      messages,
      format: "json",
      stream: false,
    }),
  });

  if (!res.ok) {
    throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
  }

  const data = await res.json();
  const rawContent = data.message?.content ?? "";
  const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");

  return JSON.parse(cleaned.trim());
}

// ──────────────────────────────────────────────
// Fixed human-reference ground truth (pre-written)
// ──────────────────────────────────────────────

const CASES = [
  {
    id: "Case 1 — Original 54N Failure (Staff Capacity vs Supplier Lead Time)",
    problem: "Orders are arriving late and customers have started complaining.",
    hypothesisA: "Delivery delays are being caused by insufficient staff capacity.",
    hypothesisB: "Delivery delays are being caused by unreliable supplier lead times.",
    reference: {
      changesInformationNeededNext: true,
      evidenceForA_semantic: ["staffing", "capacity", "workload/throughput"],
      evidenceForB_semantic: ["supplier lead times", "delivery reliability"]
    },
  },
  {
    id: "Case 2 — Same Cause, Paraphrased (Control)",
    problem: "Orders are arriving late and customers have started complaining.",
    hypothesisA: "The team may not have enough capacity to process orders on time.",
    hypothesisB: "Insufficient staff capacity may be causing the order delays.",
    reference: {
      changesInformationNeededNext: false,
      evidenceForA_semantic: ["staff capacity", "workload", "processing throughput"],
      evidenceForB_semantic: ["staff capacity", "workload", "processing throughput"]
    },
  },
  {
    id: "Case 3 — Different Causes, Different Domain (Pricing vs Checkout)",
    problem: "Website sales have fallen sharply over the last month.",
    hypothesisA: "The fall may be caused by a recent increase in product prices.",
    hypothesisB: "The fall may be caused by a technical checkout problem.",
    reference: {
      changesInformationNeededNext: true,
      evidenceForA_semantic: ["pricing", "conversion/customer response"],
      evidenceForB_semantic: ["checkout errors", "technical/funnel evidence"]
    },
  },
];

// ──────────────────────────────────────────────
// Semantic evaluation against fixed human references
// ──────────────────────────────────────────────

function evaluateEvidence(modelResult, caseType) {
  const result = modelResult;
  let issues = [];
  let notes = [];

  // Must provide evidence lists
  if (!Array.isArray(result.evidenceForA) || result.evidenceForA.length === 0) {
    issues.push("missing_evidence_A: evidenceForA is empty or not an array");
  }
  if (!Array.isArray(result.evidenceForB) || result.evidenceForB.length === 0) {
    issues.push("missing_evidence_B: evidenceForB is empty or not an array");
  }

  // Check invariant violations (no winner, no next question)
  const allText = JSON.stringify(result).toLowerCase();
  const hasWinnerSelection = allText.includes("preferred") ||
    allText.includes("more likely") ||
    allText.includes("should go with") ||
    allText.includes("is the winner");

  // Manual semantic judge per case type
  const evA = result.evidenceForA?.join(" ").toLowerCase() ?? "";
  const evB = result.evidenceForB?.join(" ").toLowerCase() ?? "";

  let evidence_correct = true;
  let evidence_details = [];

  if (caseType === "different_causes") {
    // Case 1: A should be staffing/capacity, B should be supplier/lead times
    const hasStaffingKeywords = ["staff", "capacity", "workload", "throughput", "personnel", "headcount", "roster"].some(k => evA.includes(k));
    const hasSupplierKeywords = ["supplier", "lead time", "delivery reliability", "vendor", "supply chain", "procurement", "fulfillment partner"].some(k => evB.includes(k));

    if (!hasStaffingKeywords) {
      evidence_correct = false;
      evidence_details.push("evidence_A_lacks_staffing/capacity focus");
    }
    if (!hasSupplierKeywords) {
      evidence_correct = false;
      evidence_details.push("evidence_B_lacks_supplier/lead-time focus");
    }

    // Check that they are distinct (not overlapping heavily on same domain)
    const staffInBoth = ["staff", "capacity"].some(k => evA.includes(k) && evB.includes(k));
    if (staffInBoth) {
      evidence_details.push("WARNING: staffing term appears in both evidence sets — may indicate conflation");
    }
  }

  if (caseType === "same_cause_paraphrase") {
    // Case 2: Both should reference the same domain (staff capacity)
    const evA_words = new Set(evA.split(/\s+/));
    const evB_words = new Set(evB.split(/\s+/));
    const overlapCount = [...evA_words].filter(w => evB_words.has(w)).length;

    if (overlapCount < 3) {
      evidence_correct = false;
      evidence_details.push("evidence_paraphrase: too few overlapping terms — may not be materially equivalent");
    } else {
      evidence_details.push(`evidence_paraphrase: ${overlapCount} overlapping terms suggests material equivalence`);
    }

    // Both should mention staffing/capacity
    const hasStaffing = ["staff", "capacity"].every(k => evA.includes(k) && evB.includes(k));
    if (!hasStaffing) {
      evidence_details.push("evidence_paraphrase: staffing/capacity concept not present in both sets");
    }
  }

  if (caseType === "different_domains") {
    // Case 3: A should be pricing, B should be checkout/technical
    const hasPricingKeywords = ["price", "pricing", "cost", "charge", "monetary"].some(k => evA.includes(k));
    const hasCheckoutKeywords = ["checkout", "cart", "payment", "funnel", "technical", "error", "browser", "device", "load time"].some(k => evB.includes(k));

    if (!hasPricingKeywords) {
      evidence_correct = false;
      evidence_details.push("evidence_A_lacks_pricing focus");
    }
    if (!hasCheckoutKeywords) {
      evidence_correct = false;
      evidence_details.push("evidence_B_lacks_checkout/technical focus");
    }
  }

  return {
    evidence_correct,
    evidence_failed: !evidence_correct,
    evidence_details,
    notes,
    issues,
    has_winner_selection: hasWinnerSelection,
    raw: result,
  };
}

function evaluateConsequence(modelResult, reference) {
  const result = modelResult;
  let issues = [];
  let notes = [];

  if (result.changesInformationNeededNext !== reference.changesInformationNeededNext) {
    issues.push(`consequence_mismatch: expected ${reference.changesInformationNeededNext}, got ${result.changesInformationNeededNext}`);
  }

  if (!result.reason || typeof result.reason !== "string") {
    issues.push("missing_reason: no reason field provided");
  }

  return {
    consequence_correct: issues.length === 0,
    consequence_failed: issues.length > 0,
    issues,
    notes,
  };
}

// ──────────────────────────────────────────────
// Test suite
// ──────────────────────────────────────────────

describe("Experiment 54Q — Structured Evidence plus Consequence in One Call", () => {
  const results = [];
  const timings = [];

  for (const c of CASES) {
    it(c.id, async () => {
      const start = Date.now();
      const result = await callStructuredEvidenceConsequence(c.problem, c.hypothesisA, c.hypothesisB);
      const elapsed = Date.now() - start;
      timings.push({ caseId: c.id, ms: elapsed });

      let caseType;
      if (c.id.includes("Case 1")) caseType = "different_causes";
      else if (c.id.includes("Case 2")) caseType = "same_cause_paraphrase";
      else caseType = "different_domains";

      let evidenceEval = evaluateEvidence(result, caseType);
      let consequenceEval = evaluateConsequence(result, c.reference);

      results.push({
        case: c,
        modelResult: result,
        evidence: evidenceEval,
        consequence: consequenceEval,
        timingMs: elapsed,
      });

      // Structural assertions (invariants)
      expect(result.evidenceForA).toBeDefined();
      expect(result.evidenceForB).toBeDefined();
      expect(Array.isArray(result.evidenceForA)).toBe(true);
      expect(Array.isArray(result.evidenceForB)).toBe(true);
      expect(typeof result.changesInformationNeededNext).toBe("boolean");
      expect(result.reason).toBeDefined();

      // No winner selection invariant
      const allText = JSON.stringify(result).toLowerCase();
      expect(allText).not.toContain("preferred");
      expect(allText).not.toContain("more likely");

      // Boolean must match reference
      expect(result.changesInformationNeededNext).toBe(c.reference.changesInformationNeededNext);
    }, 120000);
  }

  it("Experiment 54Q: aggregate results", () => {
    const evidenceCorrect = results.filter(r => r.evidence.evidence_correct).length;
    const evidenceFailed = results.filter(r => r.evidence.evidence_failed).length;
    const consequenceCorrect = results.filter(r => r.consequence.consequence_correct).length;
    const consequenceFailed = results.filter(r => r.consequence.consequence_failed).length;

    // Report findings to console for manual review
    console.log("\n=== Experiment 54Q Results ===");
    for (const r of results) {
      console.log(`\n--- ${r.case.id} ---`);
      console.log("Evidence A:", JSON.stringify(r.modelResult.evidenceForA));
      console.log("Evidence B:", JSON.stringify(r.modelResult.evidenceForB));
      console.log("Consequence:", r.modelResult.changesInformationNeededNext);
      console.log("Reason:", r.modelResult.reason);
      console.log("Evidence correct:", r.evidence.evidence_correct, r.evidence.evidence_details);
      console.log("Consequence correct:", r.consequence.consequence_correct, r.consequence.issues);
    }
    console.log(`\nEvidence-correct: ${evidenceCorrect}/${results.length}`);
    console.log(`Evidence-failed: ${evidenceFailed}/${results.length}`);
    console.log(`Consequence-correct: ${consequenceCorrect}/${results.length}`);
    console.log(`Consequence-failed: ${consequenceFailed}/${results.length}`);

    const totalMs = timings.reduce((s, t) => s + t.ms, 0);
    console.log(`Total time: ${totalMs}ms`);
    console.log(`Average time: ${(totalMs / timings.length).toFixed(1)}ms per call`);
    console.log(`Fastest: ${Math.min(...timings.map(t => t.ms))}ms`);
    console.log(`Slowest: ${Math.max(...timings.map(t => t.ms))}ms`);

    // Critical case assertion: Case 1 must get consequence correct (the original 54N failure)
    const case1 = results.find(r => r.case.id.includes("Case 1"));
    expect(case1.consequence.consequence_correct).toBe(true);

    // No winner selection in any result
    for (const r of results) {
      expect(r.evidence.has_winner_selection).toBe(false);
    }
  });
});