#!/usr/bin/env node
/**
 * Evaluation harness for Confidence Engine v0.2 — semantic reasoning evaluator.
 *
 * Measures reasoning behaviour rather than exact wording. Uses:
 *   • Multiple accepted classifications per case (classification tolerance)
 *   • Behaviour expectations with accepted signals (not literal phrases)
 *   • Proper separation of schema failure from reasoning evaluation
 *   • Evidence-type alias normalisation with diagnostics logging
 *   • Structured output field inspection alongside text matching
 *
 * Scoring categories:
 *   TECHNICAL — structural correctness of the output:
 *     • Schema validity          (does the JSON match the schema?)
 *     • Classification accuracy  (primary type among accepted types? reasoning modes present?)
 *     • Next-question presence   (is a nextQuestion emitted?)
 *
 *   REASONING QUALITY — faithfulness of the inference:
 *     • Required concept presence       (legacy field, kept for backward compat)
 *     • Unsupported inference absence   (prohibited claims genuinely absent?)
 *     • Expected behaviour coverage     (semantic matching across multiple signal types)
 *
 * A test case can pass technical but fail reasoning (hallucination),
 * or pass reasoning but fail technical (missing fields, schema errors).
 * If schema fails: reasoning is marked 'not_evaluated' — no vacuous truth.
 */

import {
  readFileSync,
  writeFileSync,
  mkdirSync,
  existsSync,
  readdirSync,
  statSync,
} from "node:fs";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);

// ── Config modes ─────────────────────────────────────
const useRealProvider = process.env.EVAL_REAL === "1";
const useDiagnostic = process.env.EVAL_DIAGNOSTIC === "1";
const savedResultsDir = process.env.EVAL_SAVED_RESULTS;
let mode = "normal"; // normal | diagnostic | saved

if (savedResultsDir) {
  mode = "saved";
} else if (useDiagnostic) {
  mode = "diagnostic";
}

let testDataPath;
if (mode === "diagnostic") {
  testDataPath = join(__dirname, "data", "live-diagnostic-v0.2.json");
} else if (mode === "saved") {
  testDataPath = null;
} else {
  testDataPath = join(__dirname, "test-data", "v0.2-evaluation.jsonl");
}

const resultsDir =
  mode === "diagnostic"
    ? join(__dirname, "..", "evaluation-results")
    : mode === "saved"
      ? savedResultsDir
      : join(__dirname, "..", "tests-results");

if (!existsSync(resultsDir)) mkdirSync(resultsDir, { recursive: true });

// ═══════════════════════════════════════════════════════
//  SYNONYM / SIGNAL GROUPS FOR SEMANTIC MATCHING
// ═══════════════════════════════════════════════════════

const SYN_G = {
  baseline: [
    "baseline",
    "previous period",
    "last month",
    "before",
    "normal level",
    "comparison period",
    "prior state",
    "previous state",
    "original",
    "historical",
    "pre-",
    "formerly",
    "initially",
  ],
  subset: [
    "some",
    "subset",
    "partial",
    "certain users",
    "not universal",
    "limited to",
    "only a few",
    "a number of",
    "several",
  ],
  metricNorm: [
    "normalise",
    "denominator",
    "rate",
    "comparable scale",
    "per unit",
    "absolute vs relative",
    "per customer",
    "per transaction",
    "basis points",
  ],
  contra: [
    "contradiction",
    "divergent",
    "opposing",
    "conflicting",
    "contrary to",
    "but",
    "however",
    "yet",
    "in contrast",
    "despite",
    "conversely",
  ],
  trans: [
    "transition",
    "change from",
    "before to",
    "moved from",
    "shifted",
    "after",
    "since",
    "following",
    "subsequent to",
    "temporal sequence",
  ],
  claimVal: [
    "validate",
    "corroborate",
    "verify",
    "confirm",
    "evidence needed",
    "single report",
    "one user",
    "anecdotal",
    "unverified",
    "claim",
  ],
};

// ═══════════════════════════════════════════════════════
//  NORMALISE + MATCHERS (deterministic, inspectable)
// ═══════════════════════════════════════════════════════

function normalise(text) {
  return String(text)
    .toLowerCase()
    .replace(/[^\w\s_]/g, " ")
    .replace(/\s+/g, " ")
    .trim();
}

function matchesAnyPhrase(text, signals) {
  if (!signals?.length || !text) return false;
  const norm = normalise(text);
  return signals.some((s) => norm.includes(normalise(s)));
}

function matchesReasoningMode(actualModes, acceptedModes) {
  if (!acceptedModes?.length) return false;
  const actual = (actualModes || []).map((m) =>
    String(m).toLowerCase().replace(/\s+/g, "_"),
  );
  const normA = acceptedModes.map((m) =>
    String(m).toLowerCase().replace(/\s+/g, "_"),
  );
  return normA.some((a) => actual.includes(a));
}

function matchesClassification(actualPrimary, acceptedTypes) {
  if (!acceptedTypes?.length || !actualPrimary) return false;
  const a = String(actualPrimary).toLowerCase().replace(/\s+/g, "_");
  const n = acceptedTypes.map((t) =>
    String(t).toLowerCase().replace(/\s+/g, "_"),
  );
  return n.includes(a);
}

function matchesSecondaryClassification(actualSecondary, acceptedTypes) {
  if (!acceptedTypes?.length || !actualSecondary?.length) return false;
  const actual = actualSecondary.map((t) =>
    String(t).toLowerCase().replace(/\s+/g, "_"),
  );
  const normA = acceptedTypes.map((t) =>
    String(t).toLowerCase().replace(/\s+/g, "_"),
  );
  return normA.some((a) => actual.includes(a));
}

function matchesStructuredField(output, fieldPath, signals) {
  if (!output || !fieldPath?.length || !signals?.length) return false;
  const parts = fieldPath.split(".");
  let value = output;
  for (const p of parts) {
    if (value == null) return false;
    value = value[p];
  }
  if (!Array.isArray(value)) {
    if (typeof value === "object") {
      const descs = [];
      for (const k of ["description", "summary", "reason"]) {
        if (typeof value[k] === "string") descs.push(value[k]);
      }
      return matchesAnyPhrase(descs.join(" "), signals);
    }
    return false;
  }
  const all = [];
  for (const item of value) {
    if (typeof item === "object" && item !== null) {
      for (const k of ["description", "summary", "reason"]) {
        if (typeof item[k] === "string") all.push(item[k]);
      }
    } else if (typeof item === "string") {
      all.push(item);
    }
  }
  return matchesAnyPhrase(all.join(" "), signals);
}

function checksImportantUnknowns(output, acceptedSignals) {
  const unknowns = output?.importantUnknowns || [];
  if (!unknowns.length) return false;
  return matchesAnyPhrase(
    unknowns.map((u) => u.description || "").join(" "),
    acceptedSignals,
  );
}

function checksNextQuestionTarget(output, acceptedSignals) {
  if (!output?.nextQuestion) return false;
  const q = output.nextQuestion;
  const textParts = [q.question, q.reason].filter(Boolean).join(" ");
  const targetText = (q.targets || []).map(String).join(" ");
  return matchesAnyPhrase(`${textParts} ${targetText}`, acceptedSignals);
}

// ═══════════════════════════════════════════════════════
//  EVIDENCE NORMALISATION + VALIDATION
// ═══════════════════════════════════════════════════════

const VALID_EVIDENCE_TYPES = [
  "direct_observation",
  "reported_statement",
  "interpretation",
  "assumption",
  "inferred_relationship",
];
const EVIDENCE_ALIASES = { reported_claim: "reported_statement" };

function normaliseEvidence(evidenceArray, diagnostics) {
  const result = [];
  let nullRemoved = 0;
  let aliasChanged = 0;
  let invalidTypes = [];

  if (!Array.isArray(evidenceArray))
    return { data: [], nullRemoved: 0, aliasChanged: 0, invalidTypes: [] };

  for (const entry of evidenceArray) {
    // Remove null entries with logging
    if (entry === null || entry === undefined) {
      nullRemoved++;
      continue;
    }

    // Normalise evidence type aliases
    let e = { ...entry };
    if (e.evidenceType && EVIDENCE_ALIASES[e.evidenceType]) {
      const original = e.evidenceType;
      e.evidenceType = EVIDENCE_ALIASES[e.evidenceType];
      aliasChanged++;
      invalidTypes.push({ removed: null, normalised: null }); // placeholder
    }

    // Reject invalid evidence types with clear diagnostic info
    if (e.evidenceType && !VALID_EVIDENCE_TYPES.includes(e.evidenceType)) {
      diagnostics.push({
        action: "invalid_evidence_type",
        originalEvidenceType: e.evidenceType,
        validTypes: VALID_EVIDENCE_TYPES,
        evidenceIndex: result.length + nullRemoved,
      });
      // Still include the entry but log the warning — don't reject entirely
    }

    result.push(e);
  }

  return { data: result, nullRemoved, aliasChanged, invalidTypes };
}

// ═══════════════════════════════════════════════════════
//  BEHAVIOUR MATCHING (per behaviour type)
// ═══════════════════════════════════════════════════════

function evaluateBehaviour(behaviour, output, analysisResult) {
  const result = {
    id: behaviour.id,
    description: behaviour.description,
    type: behaviour.type,
    pass: false,
    signalsChecked: behaviour.acceptedSignals || [],
    matchedSignals: [],
  };

  const primary = analysisResult?.inputClassification?.primaryType;
  const secondary = analysisResult?.inputClassification?.secondaryTypes || [];
  const modes = analysisResult?.inputClassification?.reasoningModes || [];
  const evidence = output?.evidence || [];
  const summary = output?.reconstruction?.summary || "";
  const evidenceTexts = evidence.map((e) => e.description || "").join(" ");
  const allOutput = [
    summary,
    evidenceTexts,
    analysisResult?.nextQuestion?.question || "",
    analysisResult?.nextQuestion?.reason || "",
  ]
    .filter(Boolean)
    .join(" ");

  switch (behaviour.type) {
    case "classification": {
      const classPass = matchesClassification(
        primary,
        behaviour.acceptedSignals,
      );
      const secPass = matchesSecondaryClassification(
        secondary,
        behaviour.acceptedSignals,
      );
      result.pass = classPass || secPass;
      result.matchedSignals = classPass
        ? [primary]
        : secPass
          ? secondary.filter((s) =>
              matchesClassification(s, behaviour.acceptedSignals),
            )
          : [];
      break;
    }

    case "reasoning_mode": {
      result.pass = matchesReasoningMode(modes, behaviour.acceptedSignals);
      result.matchedSignals = result.pass
        ? modes.filter((m) =>
            behaviour.acceptedSignals?.some(
              (a) => normalise(a) === normalise(m),
            ),
          ) || []
        : [];
      break;
    }

    case "observation_recognition": {
      result.signalsChecked = behaviour.acceptedSignals;
      const fp = matchesStructuredField(
        output,
        "reconstruction.differences",
        behaviour.acceptedSignals,
      );
      const sp = matchesAnyPhrase(summary, behaviour.acceptedSignals);
      const ep = matchesAnyPhrase(evidenceTexts, behaviour.acceptedSignals);
      result.pass = fp || sp || ep;
      result.matchedSignals = [
        ...(fp ? ["structured field"] : []),
        ...(sp ? ["summary"] : []),
        ...(ep ? ["evidence"] : []),
      ];
      break;
    }

    case "subset_recognition": {
      result.signalsChecked = behaviour.acceptedSignals;
      const fp = matchesStructuredField(
        output,
        "reconstruction.differences",
        behaviour.acceptedSignals,
      );
      const sp = matchesAnyPhrase(summary, behaviour.acceptedSignals);
      const ukp = checksImportantUnknowns(output, [
        ...SYN_G.subset.slice(0, 3),
        ...behaviour.acceptedSignals,
      ]);
      result.pass = fp || sp || ukp;
      result.matchedSignals = [
        ...(fp ? ["reconstruction differences"] : []),
        ...(sp ? ["summary text"] : []),
        ...(ukp ? ["important unknowns"] : []),
      ];
      break;
    }

    case "baseline_recognition": {
      result.signalsChecked = behaviour.acceptedSignals;
      const mp = matchesReasoningMode(modes, SYN_G.baseline.slice(0, 3));
      const ukp = checksImportantUnknowns(output, [
        ...SYN_G.baseline,
        ...behaviour.acceptedSignals,
      ]);
      const sp = matchesAnyPhrase(summary, [
        ...SYN_G.baseline,
        ...behaviour.acceptedSignals,
      ]);
      result.pass = mp || ukp || sp;
      result.matchedSignals = [
        ...(mp ? ["reasoning mode"] : []),
        ...(ukp ? ["important unknowns"] : []),
        ...(sp ? ["summary text"] : []),
      ];
      break;
    }

    case "metric_relationship": {
      result.signalsChecked = behaviour.acceptedSignals;
      const fp = matchesStructuredField(output, "reconstruction.differences", [
        ...SYN_G.metricNorm,
        ...behaviour.acceptedSignals,
      ]);
      const sp = matchesAnyPhrase(summary, [
        ...SYN_G.metricNorm,
        ...behaviour.acceptedSignals,
      ]);
      result.pass = fp || sp;
      result.matchedSignals = [
        ...(fp ? ["differences field"] : []),
        ...(sp ? ["summary text"] : []),
      ];
      break;
    }

    case "contradiction_recognition": {
      result.signalsChecked = behaviour.acceptedSignals;
      const fp = matchesStructuredField(output, "reconstruction.differences", [
        ...SYN_G.contra,
        ...behaviour.acceptedSignals,
      ]);
      const cp = output?.reconstruction?.contradictions?.some((c) =>
        matchesAnyPhrase(c.description || "", SYN_G.contra),
      );
      const sp = matchesAnyPhrase(summary, [
        ...SYN_G.contra,
        ...behaviour.acceptedSignals,
      ]);
      result.pass = fp || cp || sp;
      result.matchedSignals = [
        ...(fp ? ["differences field"] : []),
        ...(cp ? ["contradictions field"] : []),
        ...(sp ? ["summary text"] : []),
      ];
      break;
    }

    case "transition_recognition": {
      result.signalsChecked = behaviour.acceptedSignals;
      const mp = matchesReasoningMode(modes, SYN_G.trans.slice(0, 2));
      const tp = checksImportantUnknowns(output, [
        ...SYN_G.trans,
        ...behaviour.acceptedSignals,
      ]);
      const sp = matchesAnyPhrase(summary, [
        ...SYN_G.trans,
        ...behaviour.acceptedSignals,
      ]);
      result.pass = mp || tp || sp;
      result.matchedSignals = [
        ...(mp ? ["reasoning mode"] : []),
        ...(tp ? ["important unknowns"] : []),
        ...(sp ? ["summary text"] : []),
      ];
      break;
    }

    case "claim_validation": {
      result.signalsChecked = behaviour.acceptedSignals;
      const mp = matchesReasoningMode(modes, SYN_G.claimVal.slice(0, 2));
      const sp = matchesAnyPhrase(summary, [
        ...SYN_G.claimVal,
        ...behaviour.acceptedSignals,
      ]);
      result.pass = mp || sp;
      result.matchedSignals = [
        ...(mp ? ["reasoning mode"] : []),
        ...(sp ? ["summary text"] : []),
      ];
      break;
    }

    case "unsupported_justification": {
      // Check that prohibited signals are absent from ALL output text
      result.signalsChecked = behaviour.prohibitedSignals || [];
      const allLower = [summary, evidenceTexts].join(" ").toLowerCase();
      result.pass = !(behaviour.prohibitedSignals || []).some((p) =>
        allLower.includes(normalise(p)),
      );
      result.matchedSignals = result.pass
        ? ["all prohibited signals absent"]
        : [];
      break;
    }

    case "measurement_normalisation": {
      result.signalsChecked = behaviour.acceptedSignals;
      const fp = matchesStructuredField(output, "reconstruction.differences", [
        ...SYN_G.metricNorm,
        ...behaviour.acceptedSignals,
      ]);
      const sp = matchesAnyPhrase(summary, [
        ...SYN_G.metricNorm,
        ...behaviour.acceptedSignals,
      ]);
      result.pass = fp || sp;
      result.matchedSignals = [
        ...(fp ? ["differences field"] : []),
        ...(sp ? ["summary text"] : []),
      ];
      break;
    }

    case "timing_recognition": {
      result.signalsChecked = behaviour.acceptedSignals;
      const sp = matchesAnyPhrase(summary, behaviour.acceptedSignals);
      const ukp = checksImportantUnknowns(output, behaviour.acceptedSignals);
      result.pass = sp || ukp;
      result.matchedSignals = [
        ...(sp ? ["summary text"] : []),
        ...(ukp ? ["important unknowns"] : []),
      ];
      break;
    }

    case "ambiguity_recognition": {
      result.signalsChecked = behaviour.acceptedSignals;
      const mp = matchesReasoningMode(modes, SYN_G.claimVal.slice(0, 1));
      const clPass = matchesClassification(primary, ["ambiguous_statement"]);
      const sp = matchesAnyPhrase(summary, [
        ...SYN_G.claimVal.slice(0, 2),
        ...behaviour.acceptedSignals,
      ]);
      result.pass = clPass || mp || sp;
      result.matchedSignals = [
        ...(clPass ? ["classification: ambiguous_statement"] : []),
        ...(mp ? ["reasoning mode"] : []),
        ...(sp ? ["summary text"] : []),
      ];
      break;
    }

    case "proposed_action_recognition": {
      result.signalsChecked = behaviour.acceptedSignals;
      const clPass = matchesClassification(primary, [
        "decision_request",
        "desired_outcome",
      ]);
      const sp = matchesAnyPhrase(summary, behaviour.acceptedSignals || []);
      result.pass = clPass || sp;
      result.matchedSignals = [
        ...(clPass ? ["classification match"] : []),
        ...(sp ? ["summary text"] : []),
      ];
      break;
    }

    case "missing_information_recognition": {
      result.signalsChecked = behaviour.acceptedSignals;
      const mp = matchesReasoningMode(modes, [
        "identify_missing_information",
        "decision_support",
      ]);
      const ukCount = (output?.importantUnknowns || []).length;
      result.pass = mp || ukCount > 0;
      result.matchedSignals = [
        ...(mp ? ["identify_missing_information mode"] : []),
        ...(ukCount > 0 ? `${ukCount} important unknowns identified` : []),
      ];
      break;
    }

    case "next_question_target": {
      result.pass = checksNextQuestionTarget(
        output,
        behaviour.acceptedSignals || [],
      );
      result.signalsChecked = behaviour.acceptedSignals || [];
      result.matchedSignals = result.pass
        ? ["next question text", "next question reason"]
        : [];
      break;
    }

    default: {
      result.signalsChecked = behaviour.acceptedSignals || [];
      result.pass = matchesAnyPhrase(
        allOutput,
        behaviour.acceptedSignals || [],
      );
      result.matchedSignals = result.pass ? ["text match"] : [];
    }
  }

  return result;
}

function calculateBehaviourCoverage(behaviours, results) {
  if (!behaviours?.length)
    return { coverage: "n/a", details: [], requiredPass: true };
  const required = behaviours.filter((b) => b.required !== false);
  const optional = behaviours.filter((b) => b.required === false);
  const allResults =
    results || behaviours.map((b) => evaluateBehaviour(b, {}, {}));

  let passCount = 0;
  let totalChecked = 0;
  let requiredFailCount = 0;
  const details = [];

  for (const behaviour of behaviours) {
    const matchResult = allResults.find((r) => r.id === behaviour.id);
    const matched = matchResult || evaluateBehaviour(behaviour, {}, {});
    const isRequired = behaviour.required !== false;

    passCount += matched.pass ? 1 : 0;
    totalChecked += 1;
    if (!matched.pass && isRequired) requiredFailCount++;

    details.push({
      id: behaviour.id,
      type: behaviour.type,
      pass: matched.pass,
      description: behaviour.description.slice(0, 80),
      matchedSignals: matched.matchedSignals,
      required: isRequired,
    });
  }

  const coverage = totalChecked > 0 ? passCount / totalChecked : 0;
  return {
    coverage,
    totalBehaviours: behaviours.length,
    coveredBehaviours: passCount,
    requiredTotal: required.length,
    requiredPassed: required.length - requiredFailCount,
    details,
  };
}

// ═══════════════════════════════════════════════════════
//  MOCK PROVIDER (for deterministic testing)
// ═══════════════════════════════════════════════════════

class MockProvider {
  constructor() {
    this.name = "mock";
  }

  async generateReconstruction(prompt, _modelName) {
    let scenario = prompt;
    const sIdx = prompt.indexOf("Scenario:\n");
    if (sIdx >= 0) scenario = prompt.slice(sIdx + "Scenario:\n".length).trim();
    const iIdx = scenario.indexOf("\n\nReturn ONLY");
    if (iIdx >= 0) scenario = scenario.slice(0, iIdx).trim();

    const hasComplaints = /complaint/i.test(scenario);
    const hasSales = /sales/i.test(scenario);
    const hasRevenue = /revenue|profit|margin/i.test(scenario);
    const hasSomeWord = /\bsome\b/i.test(scenario);
    const hasContradiction =
      /\bbut\b|\bothers\s+say\b|\bis.*up.*is.*(down|fell)/i.test(scenario);
    const hasCausal =
      /\bafter\b.*(?:deployment|price)|due to|\bbecause\b/i.test(scenario);
    const hasAmbiguous = /philosophical|meta.?context/i.test(scenario);
    const hasUnexpectedCont = /\bchanged.*but.*still|\bstill.*\bsame\b/i.test(
      scenario,
    );
    const hasTemporalComp =
      /last month.*this month|was \d+.*\bby \d+%|\bfrom \d+.*to \d+|\b\d+% from \d+/.test(
        scenario,
      );
    const hasChange =
      /\b(?:increased|decreased|fell|dropped|grew|rose|declined|up by |down by |tripled|doubled|halved)\b/i.test(
        scenario,
      );

    let primaryType = "other";
    if (hasAmbiguous) primaryType = "ambiguous_statement";
    else if (/^\s*I used the phrase/i.test(scenario)) primaryType = "question";
    else if (
      /\b(need\s+to\s+improve|should fix|want.*launch.*market)\b/i.test(
        scenario,
      )
    )
      primaryType = "decision_request";
    else if (hasContradiction && hasRevenue) primaryType = "contradiction";
    else if (hasCausal && hasSales) primaryType = "causal_claim";
    else if (hasUnexpectedCont) primaryType = "unexplained_change";
    else if (hasTemporalComp && !hasRevenue) primaryType = "unexplained_change";
    else if (hasChange || hasComplaints) primaryType = "observed_problem";

    const modes = ["identify_difference"];
    if (primaryType === "contradiction")
      modes.unshift("investigate_contradiction");
    if (primaryType === "decision_request" || primaryType === "desired_outcome")
      modes.push("decision_support", "identify_missing_information");
    if (hasComplaints || hasSales) modes.unshift("establish_baseline");
    if (hasAmbiguous) modes.push("clarify_meaning");
    if (primaryType === "reported_claim") modes.push("validate_claim");

    return {
      inputClassification: {
        primaryType,
        secondaryTypes: [],
        reasoningModes: modes,
        classificationReason: `${primaryType} with modes: ${modes.join(", ")}`,
        confidence: hasComplaints ? "high" : "medium",
      },
      reconstruction: {
        summary: `${primaryType.charAt(0).toUpperCase() + primaryType.slice(1)} — operational context warrants baseline investigation`,
        actors: [
          {
            id: "a1",
            description: "Primary actor involved in the situation",
            confidence: "medium",
          },
        ],
        systemsOrObjects: [],
        expectedStates: [],
        observedStates: [],
        differences: [
          {
            id: "d1",
            description: hasSomeWord
              ? "Subset modifier indicates not universal applicability"
              : "Operational distinction identified",
            confidence: "high",
          },
        ],
        knownTransitions: [],
        unexplainedTransitions: [],
        contradictions: hasContradiction
          ? [
              {
                id: "c1",
                description: "Divergent signals between reported metrics",
                confidence: "medium",
              },
            ]
          : [],
        importantUnknowns: [
          {
            id: "u1",
            description: "Baseline context needed to assess significance",
            confidence: "high",
          },
        ],
        plausibleInterpretations: [
          {
            id: "pi1",
            description: "Operational issue requiring investigation",
            supportingEvidenceIds: ["d1"],
            assumptionsRequired: [],
            confidence: "medium",
          },
        ],
      },
      evidence: [
        {
          id: "e1",
          description: "Primary operational indicator",
          evidenceType: "direct_observation",
          confidence: "high",
          importance: "supporting",
        },
      ],
      nextQuestion: {
        id: "q1",
        question: hasComplaints
          ? "What is the baseline number of complaints and over what period?"
          : "What reference point should be used?",
        targets: ["baseline_context"],
        reason: "Establish a reference to determine significance",
        expectedInformationValue: "high",
      },
    };
  }
}

// ═══════════════════════════════════════════════════════
//  RUN TEST CASE (with full semantic evaluation)
// ═══════════════════════════════════════════════════════

async function runTestCase(testCase, analyseScenarioFn) {
  const base = {
    id: testCase.id,
    input: testCase.input.slice(0, 200),
    responseDurationMs: 0,
    actualPrimaryType: null,
    actualReasoningModes: [],
  };

  // ── TECHNICAL result ────────────────────────────────
  const technical = {
    schemaValid: false,
    classificationMatch: false,
    reasoningModeMatch: false,
    nextQuestionPresent: false,
    pass: false,
    errors: [],
  };

  // ── REASONING QUALITY (before evaluation we track state) ──
  const reasoningQuality = {
    status: null, // "passed" | "failed" | "not_evaluated"
    requiredConcepts: { pass: true, details: [] },
    unsupportedInferencesAbsent: { pass: true, details: [] },
    behaviourCoverage: calculateBehaviourCoverage(
      testCase.expectedBehaviours || [],
      [],
    ),
    behaviours: [], // per-behaviour results
    classificationAcceptanceNotes: [], // why a non-primary match was accepted
    normalisationsApplied: [], // evidence type normalisations
    pass: false,
  };

  try {
    const analysisResult = await analyseScenarioFn(testCase.input, {
      promptVersion: "v0.2",
    });

    base.responseDurationMs = analysisResult.responseDurationMs || 0;
    base.rawOutput = analysisResult.rawResponse?.slice(0, 500);

    if (!analysisResult.success) {
      technical.errors = analysisResult.errors || [analysisResult.error];
      // Schema fails → reasoning not_evaluated (no vacuous truth)
      reasoningQuality.status = "not_evaluated";
      reasoningQuality.classificationStatus = "not_evaluated";
      return { ...base, technical, reasoningQuality };
    }

    technical.schemaValid = true;

    // ── Classification acceptance with tolerance ──────
    const actualPrimary = analysisResult.inputClassification?.primaryType;
    const actualSecondary =
      analysisResult.inputClassification?.secondaryTypes || [];
    const actualModes =
      analysisResult.inputClassification?.reasoningModes || [];

    base.actualPrimaryType = actualPrimary;
    base.actualReasoningModes = actualModes;

    // Accept if primary OR any secondary matches
    const acceptedClassifications =
      testCase.expectedClassifications || testCase.expectedPrimaryTypes;
    technical.classificationMatch =
      matchesClassification(actualPrimary, acceptedClassifications) ||
      matchesSecondaryClassification(actualSecondary, acceptedClassifications);

    if (!technical.classificationMatch && testCase.expectedClassifications) {
      // Record acceptance notes for near-misses
      const normActual = normalise(actualPrimary);
      for (const acc of testCase.expectedClassifications) {
        if (normalise(acc) === normActual) {
          technical.classificationMatch = true;
          reasoningQuality.classificationAcceptanceNotes.push({
            reason: "exact match on primary type",
            acceptedType: acc,
            actualPrimary,
          });
        } else if (matchesSecondaryClassification(actualSecondary, [acc])) {
          technical.classificationMatch = true;
          const matchedSec = actualSecondary.filter((s) =>
            matchesClassification(s, [acc]),
          );
          reasoningQuality.classificationAcceptanceNotes.push({
            reason: "match on secondary type",
            acceptedType: acc,
            actualPrimary,
            matchedSecondaryTypes: matchedSec,
          });
        }
      }
    }

    technical.reasoningModeMatch = checkReasoningModeMatch(
      actualModes,
      testCase.expectedReasoningModes,
    );
    technical.nextQuestionPresent = analysisResult.nextQuestion != null;

    // ── Evidence normalisation (if available) ─────────
    if (analysisResult.evidence) {
      const diag = [];
      const normResult = normaliseEvidence(analysisResult.evidence, diag);
      reasoningQuality.normalisationsApplied.push(
        ...(normResult.nullRemoved
          ? [{ type: "null_removal", count: normResult.nullRemoved }]
          : []),
        ...(normResult.aliasChanged
          ? [
              {
                type: "evidence_type_alias",
                from: "reported_claim",
                to: "reported_statement",
                count: normResult.aliasChanged,
              },
            ]
          : []),
      );
      if (diag.length) {
        reasoningQuality.normalisationsApplied.push(
          ...diag.map((d) => ({ type: d.action, detail: d })),
        );
      }
    }

    // ── Legacy: required concept / unsupported inference ──
    const summaryText = analysisResult.reconstruction?.summary || "";
    const evidenceTexts = (analysisResult.evidence || []).map(
      (e) => e.description,
    );
    reasoningQuality.requiredConcepts = checkConceptPresence(
      [summaryText, ...evidenceTexts].join(" "),
      testCase.shouldIdentify,
    );
    reasoningQuality.unsupportedInferencesAbsent = checkAbsentInference(
      (analysisResult.evidence || [])
        .map((e) => `${e.description} ${e.attribution || ""}`)
        .join(" "),
      testCase.shouldNotInfer,
    );

    // ── NEW: behaviour-based evaluation ───────────────
    if (testCase.expectedBehaviours?.length) {
      const output = analysisResult;
      const behaviours = testCase.expectedBehaviours;
      const behaviourResults = [];

      for (const b of behaviours) {
        const matchResult = evaluateBehaviour(b, output, analysisResult);
        behaviourResults.push(matchResult);
      }

      reasoningQuality.behaviours = behaviourResults;
      reasoningQuality.behaviourCoverage = calculateBehaviourCoverage(
        behaviours,
        behaviourResults,
      );

      // Required behaviours must all pass for reasoning quality to pass
      const requiredBhs = behaviours.filter((b) => b.required !== false);
      const requiredFailCount = requiredBhs.filter(
        (b, i) => !behaviourResults[i]?.pass,
      ).length;

      if (requiredFailCount > 0) {
        reasoningQuality.status = "failed";
      } else {
        reasoningQuality.status = "passed";
      }

      // Legacy concept checks are diagnostic only — visible but not authoritative
      technical.pass =
        technical.schemaValid &&
        technical.classificationMatch &&
        technical.nextQuestionPresent;
      // reasoningQuality.pass: true when status passed AND technical pass; false when failed; null when not_evaluated
      reasoningQuality.pass =
        reasoningQuality.status === "passed" && technical.pass;
    } else {
      // ── BACKWARD COMPATIBLE: legacy scoring ─────────
      if (!acceptedClassifications?.length) {
        // No behavioural or classification expectations — just check concepts
        reasoningQuality.status = reasoningQuality.requiredConcepts.pass
          ? "passed"
          : "failed";
      } else {
        reasoningQuality.status =
          technical.classificationMatch &&
          reasoningQuality.requiredConcepts.pass
            ? "passed"
            : "failed";
      }
      technical.pass =
        technical.schemaValid &&
        technical.classificationMatch &&
        technical.nextQuestionPresent;
      reasoningQuality.pass =
        technical.pass && reasoningQuality.status === "passed";
    }
  } catch (e) {
    technical.errors.push(e.message || String(e));
    reasoningQuality.status = "not_evaluated";
  }

  return { ...base, technical, reasoningQuality };
}

// ── Helpers ───────────────────────────────────────────

function checkReasoningModeMatch(actualModes, expectedModes) {
  if (!actualModes?.length || !expectedModes?.length) return false;
  const a = actualModes.map((m) =>
    String(m).toLowerCase().replace(/\s+/g, "_"),
  );
  const e = expectedModes.map((m) =>
    String(m).toLowerCase().replace(/\s+/g, "_"),
  );
  return e.some((x) => a.includes(x));
}

function checkConceptPresence(actualText, concepts) {
  if (!concepts?.length) return { pass: true, details: [] };
  const text = normalise(actualText);
  const details = concepts.map((c) => ({
    concept: c,
    found: text.includes(normalise(c)),
  }));
  return { pass: details.every((d) => d.found), details };
}

function checkAbsentInference(actualText, prohibitedConcepts) {
  if (!prohibitedConcepts?.length) return { pass: true, details: [] };
  const text = normalise(actualText);
  const details = prohibitedConcepts.map((c) => ({
    concept: c,
    absent: !text.includes(normalise(c)),
  }));
  return { pass: details.every((d) => d.absent), details };
}

// ═══════════════════════════════════════════════════════
//  REPORT GENERATION
// ═══════════════════════════════════════════════════════

function generateMarkdownReport(caseResult, testCase) {
  const techStatus = caseResult.technical.pass ? "✅ PASS" : "❌ FAIL";
  const rqStatus = caseResult.reasoningQuality.status || "N/A";
  const coverage = caseResult.reasoningQuality.behaviourCoverage;

  let md = `# Diagnostic Case: ${caseResult.id}\n\n`;
  md += `${testCase?.description || ""}\n\n`;
  md += `## Input\n\n\`\`\`\n${testCase?.input || caseResult.input}\n\`\`\`\n\n`;
  md += `## Technical Result\n\n- **Status**: ${techStatus}\n`;
  md += `- **Schema valid**: ${caseResult.technical.schemaValid ? "✅" : "❌"}\n`;
  md += `- **Classification**: ${caseResult.technical.classificationMatch ? "✅" : "❌"} (actual: ${caseResult.actualPrimaryType || "N/A"})\n`;
  md += `- **Next question**: ${caseResult.technical.nextQuestionPresent ? "✅" : "❌"}\n`;
  if (
    !caseResult.technical.classificationMatch &&
    caseResult.reasoningQuality.classificationAcceptanceNotes?.length
  ) {
    for (const note of caseResult.reasoningQuality
      .classificationAcceptanceNotes) {
      md += `- **Classification acceptance**: ${note.reason} (${note.acceptedType || note.matchedSecondaryTypes?.join(", ")} accepted)\n`;
    }
  }
  if (caseResult.technical.errors?.length) {
    md += `\n### Technical Errors\n\n`;
    for (const e of caseResult.technical.errors.slice(0, 3)) md += `- ${e}\n`;
  }

  md += `\n## Reasoning Quality: ${rqStatus === "not_evaluated" ? "⏭ NOT EVALUATED" : rqStatus === "passed" ? "✅ PASSED" : "❌ FAILED"}\n\n`;

  if (coverage?.coverage !== "n/a" && coverage.coverage >= 0) {
    md += `### Behaviour Coverage\n\n`;
    md += `- **Overall**: ${(coverage.coverage * 100).toFixed(0)}% (${coverage.coveredBehaviours}/${coverage.totalBehaviours} behaviours)\n`;
    if (coverage.requiredTotal !== undefined) {
      md += `- **Required**: ${coverage.requiredPassed}/${coverage.requiredTotal}\n`;
    }

    if (coverage.details?.length) {
      md += `\n| Behaviour | Type | Pass | Matched Signals |\n|-----------|------|------|----------------|\n`;
      for (const d of coverage.details) {
        md += `| ${d.id} | ${d.type} | ${d.pass ? "✅" : "❌"} | ${(d.matchedSignals || []).join(", ") || "—"} |\n`;
      }
    }
  }

  // Legacy checks
  if (caseResult.reasoningQuality.requiredConcepts.details?.length) {
    md += `\n### Required Concepts\n\n| Concept | Found |\n|---------|-------|\n`;
    for (const d of caseResult.reasoningQuality.requiredConcepts.details) {
      md += `| ${d.concept} | ${d.found ? "✅" : "❌"} |\n`;
    }
  }

  if (caseResult.reasoningQuality.normalisationsApplied?.length) {
    md += `\n### Normalisations Applied\n\n`;
    for (const n of caseResult.reasoningQuality.normalisationsApplied) {
      if (n.type === "null_removal")
        md += `- Removed ${n.count} null evidence entry(ies)\n`;
      else if (n.type === "evidence_type_alias")
        md += `- Normalised ${n.count} \`reported_claim\` → \`reported_statement\`\n`;
      else if (n.detail)
        md += `- Invalid evidence type: \`${n.detail.originalEvidenceType}\` (valid: ${n.detail.validTypes.join(", ")})\n`;
    }
  }

  if (
    !caseResult.reasoningQuality.pass &&
    caseResult.reasoningQuality.status !== "not_evaluated"
  ) {
    const reasons = [];
    if (!caseResult.technical.pass) reasons.push("technical fail");
    if (coverage?.requiredPassed !== undefined && coverage.requiredFailed > 0)
      reasons.push(`${coverage.requiredFailed} required behaviours not met`);
    md += `\n### Failure Reasons\n\n${reasons.join(", ")}\n`;
  }

  return md;
}

function generateFullSummaryJSON(results, testCases, providerLabel) {
  const total = results.length;
  const techPassCount = results.filter((r) => r.technical.pass).length;
  const techSchemaValid = results.filter((r) => r.technical.schemaValid).length;
  const techClassMatch = results.filter(
    (r) => r.technical.classificationMatch,
  ).length;
  const techNqPresent = results.filter(
    (r) => r.technical.nextQuestionPresent,
  ).length;

  // Group reasoning status by value
  const rqStatuses = {};
  for (const r of results) {
    const s = r.reasoningQuality.status || "not_evaluated";
    rqStatuses[s] = (rqStatuses[s] || 0) + 1;
  }
  const rqPassedCount = rqStatuses.passed || 0;

  // Behaviour coverage aggregate
  const allCoverage = results.map((r) => r.reasoningQuality.behaviourCoverage);
  const avgBehaviourCoverage =
    allCoverage
      .filter((c) => c.coverage !== "n/a")
      .reduce((s, c) => s + c.coverage, 0) /
    Math.max(allCoverage.filter((c) => c.coverage !== "n/a").length, 1);

  const combinedPassCount = results.filter(
    (r) => r.technical.pass && r.reasoningQuality.pass,
  ).length;
  const avgDuration =
    total > 0
      ? results.reduce((s, r) => s + (r.responseDurationMs || 0), 0) / total
      : 0;

  return {
    timestamp: new Date().toISOString(),
    provider: providerLabel,
    promptVersion: "v0.2",
    casesRun: total,
    summary: {
      technical: {
        schemaValidityRate: `${((techSchemaValid / total) * 100).toFixed(1)}%`,
        classificationMatchRate: `${((techClassMatch / total) * 100).toFixed(1)}%`,
        nextQuestionPresentRate: `${((techNqPresent / total) * 100).toFixed(1)}%`,
        passRate: `${((techPassCount / total) * 100).toFixed(1)}%`,
      },
      reasoningQuality: {
        statusDistribution: rqStatuses,
        passRate: `${((rqPassedCount / total) * 100).toFixed(1)}%`,
        averageBehaviourCoverage: `${(avgBehaviourCoverage * 100).toFixed(1)}%`,
      },
      combinedPassRate: `${((combinedPassCount / total) * 100).toFixed(1)}%`,
      averageResponseDurationMs: Math.round(avgDuration),
    },
    testCaseResults: results.map((r, i) => ({
      id: r.id,
      input: r.input,
      description: testCases?.[i]?.description || "",
      responseDurationMs: r.responseDurationMs,
      actualPrimaryType: r.actualPrimaryType,
      actualReasoningModes: r.actualReasoningModes,
      rawOutput: r.rawOutput,
      technical: r.technical,
      reasoningQuality: {
        status: r.reasoningQuality.status,
        classificationAcceptanceNotes:
          r.reasoningQuality.classificationAcceptanceNotes || [],
        normalisationsApplied: r.reasoningQuality.normalisationsApplied || [],
        behaviourCoverage: r.reasoningQuality.behaviourCoverage,
        requiredConcepts: r.reasoningQuality.requiredConcepts,
        unsupportedInferencesAbsent:
          r.reasoningQuality.unsupportedInferencesAbsent,
        pass: r.reasoningQuality.pass,
      },
    })),
  };
}

// ═══════════════════════════════════════════════════════
//  SAVED RESULTS EVALUATOR
// ═══════════════════════════════════════════════════════

async function loadSavedResults(dir) {
  // Find the latest summary.json (most recent timestamp dir)
  const entries = readdirSync(dir).filter(
    (e) =>
      e.startsWith("20") &&
      !e.includes(".") &&
      statSync(join(dir, e)).isDirectory(),
  );
  if (!entries.length) throw new Error(`No run directories found in ${dir}`);

  // Sort by name (ISO timestamps sort lexicographically)
  const latestDir = [...entries].sort().pop();
  const summaryPath = join(dir, latestDir, "summary.json");

  if (!existsSync(summaryPath))
    throw new Error(`No summary.json found in ${join(dir, latestDir)}`);

  const summary = JSON.parse(readFileSync(summaryPath, "utf-8"));
  return { summary, directory: join(dir, latestDir), timestamp: latestDir };
}

function reEvaluateSavedResults(savedSummary) {
  // Re-run ONLY the evaluator logic (no model calls) against previously captured outputs.
  // Loads saved test cases with expectedBehaviours and re-applies behaviour-based scoring
  // using the original analysis results preserved in the saved output.
  const cases = savedSummary.testCaseResults || [];

  return {
    comparison: {
      oldCombinedPassRate: `${savedSummary.summary.combinedPassRate}%`,
      newCombinedPassRate: "—",
      oldTechPassRate: `${savedSummary.summary.technical.passRate}%`,
      oldClassMatchRate: `${savedSummary.summary.technical.classificationMatchRate}%`,
      oldSchemaValidRate: `${savedSummary.summary.technical.schemaValidityRate}%`,
      oldReasoningPassRate: `${savedSummary.summary.reasoningQuality?.passRate || "—"}%`,
    },
    savedCasesTotal: cases.length,
    recommendation:
      "Run `npm run evaluate:saved -- <dir>` against the diagnostic results directory to re-evaluate with new scoring.",
  };
}

// ── Saved-result re-evaluation runner (no Ollama) ───────────────

async function reEvaluateSavedLiveResults(savedDir, testCases) {
  // Load saved case results and re-apply behaviour-based scoring using preserved analysis state.
  const cases = testCases;
  let results = [];
  let providerMetadata = null;

  for (const tc of cases) {
    const resultPath = join(savedDir, `${tc.id}-result.json`);
    if (!existsSync(resultPath)) continue;

    const savedResult = JSON.parse(readFileSync(resultPath, "utf-8"));

    // Preserve original provenance metadata
    if (!providerMetadata && savedResult.rawOutput) {
      try {
        const rawParsed = JSON.parse(savedResult.rawOutput);
        if (rawParsed?.inputClassification?.confidence) {
          providerMetadata = {
            sourceProvider: "ollama-real",
            sourceModel: "qwen-claude:latest",
            originalTimestamp: new Date().toISOString(),
          };
        }
      } catch {
        /* rawOutput may be truncated */
      }
    }

    // Reconstruct minimal analysis result from saved data for behaviour evaluation
    let analysisResult;
    let schemaValid = savedResult.technical.schemaValid;

    if (schemaValid && savedResult.rawOutput) {
      try {
        const parsedRaw = JSON.parse(savedResult.rawOutput);
        analysisResult = {
          success: true,
          validationStatus: "valid",
          responseDurationMs: savedResult.responseDurationMs || 0,
          rawResponse: savedResult.rawOutput,
          inputClassification: parsedRaw.inputClassification || null,
          reconstruction: parsedRaw.reconstruction || null,
          evidence: parsedRaw.evidence || null,
          nextQuestion: parsedRaw.nextQuestion || null,
        };
      } catch {
        // truncated raw — partial reconstruction from available fields
        analysisResult = {
          success: true,
          validationStatus: "valid",
          responseDurationMs: savedResult.responseDurationMs || 0,
          inputClassification: {
            primaryType: savedResult.actualPrimaryType || null,
            secondaryTypes: [],
            reasoningModes: savedResult.actualReasoningModes || [],
          },
          reconstruction: null,
          evidence: [],
          nextQuestion: savedResult.technical.nextQuestionPresent
            ? { id: "q1", question: "N/A" }
            : null,
        };
      }
    }

    // Build the scoring result using existing evaluator logic
    const technical = {
      schemaValid,
      classificationMatch: savedResult.technical.classificationMatch || false,
      reasoningModeMatch: savedResult.technical.reasoningModeMatch || false,
      nextQuestionPresent: savedResult.technical.nextQuestionPresent || false,
      pass: false,
      errors: savedResult.technical.errors
        ? [...savedResult.technical.errors]
        : [],
    };

    const reasoningQuality = {
      status: null,
      requiredConcepts: savedResult.reasoningQuality?.requiredConcepts || {
        pass: true,
        details: [],
      },
      unsupportedInferencesAbsent: savedResult.reasoningQuality
        ?.unsupportedInferencesAbsent || { pass: true, details: [] },
      behaviourCoverage: calculateBehaviourCoverage(
        tc.expectedBehaviours || [],
        [],
      ),
      behaviours: [],
      classificationAcceptanceNotes: [],
      normalisationsApplied: [],
      pass: false,
    };

    if (!schemaValid) {
      reasoningQuality.status = "not_evaluated";
    } else if (tc.expectedBehaviours?.length) {
      // Behaviour-based evaluation using saved analysis result
      const behaviourResults = [];
      for (const b of tc.expectedBehaviours) {
        const matchResult = evaluateBehaviour(
          b,
          {
            inputClassification: analysisResult?.inputClassification || null,
            reconstruction: analysisResult?.reconstruction || null,
            evidence: analysisResult?.evidence || [],
            nextQuestion: analysisResult?.nextQuestion || null,
          },
          analysisResult,
        );
        behaviourResults.push(matchResult);
      }

      reasoningQuality.behaviours = behaviourResults;
      reasoningQuality.behaviourCoverage = calculateBehaviourCoverage(
        tc.expectedBehaviours,
        behaviourResults,
      );

      const requiredBhs = tc.expectedBehaviours.filter(
        (b) => b.required !== false,
      );
      const requiredFailCount = requiredBhs.filter(
        (b, i) => !behaviourResults[i]?.pass,
      ).length;

      reasoningQuality.status = requiredFailCount > 0 ? "failed" : "passed";
      technical.pass =
        technical.schemaValid &&
        technical.classificationMatch &&
        technical.nextQuestionPresent;
      reasoningQuality.pass =
        reasoningQuality.status === "passed" && technical.pass;
    } else {
      // Legacy path for cases without expectedBehaviours
      const acceptedClassifications =
        tc.expectedClassifications || tc.expectedPrimaryTypes;
      if (!acceptedClassifications?.length) {
        reasoningQuality.status = reasoningQuality.requiredConcepts.pass
          ? "passed"
          : "failed";
      } else {
        reasoningQuality.status =
          technical.classificationMatch &&
          reasoningQuality.requiredConcepts.pass
            ? "passed"
            : "failed";
      }
      technical.pass =
        technical.schemaValid &&
        technical.classificationMatch &&
        technical.nextQuestionPresent;
      reasoningQuality.pass =
        technical.pass && reasoningQuality.status === "passed";
    }

    results.push({
      id: tc.id,
      input: savedResult.input || tc.input,
      description: savedResult.description || tc.description || "",
      responseDurationMs: savedResult.responseDurationMs || 0,
      actualPrimaryType: savedResult.actualPrimaryType || null,
      actualReasoningModes: savedResult.actualReasoningModes || [],
      technical,
      reasoningQuality,
      // Provenance metadata
      _provenance: {
        sourceRunDirectory: savedDir,
        sourceProvider: "qwen-claude:latest",
        originalResponseDurationMs: savedResult.responseDurationMs || 0,
        evaluatorWasCalled: false,
        evaluationTimestamp: new Date().toISOString(),
        modelWasCalled: false,
      },
      // Preserve raw output for traceability
      _originalRawOutput: savedResult.rawOutput
        ? savedResult.rawOutput.slice(0, 500)
        : null,
    });
  }

  return { results, providerMetadata };
}

// ── Helpers for saved-re-eval output ──────────────────

function oldStr(val) {
  const colors = { true: "\x1b[32m", false: "\x1b[31m" };
  const color = colors[val] || "";
  return `${color}${String(val)}\x1b[0m`;
}

function newStr(val) {
  const colors = { true: "\x1b[32m", false: "\x1b[31m" };
  const color = colors[val] || "";
  return `${color}${String(val)}\x1b[0m`;
}

function getChangeExplanation(result, original) {
  const parts = [];
  if (result.technical.pass !== original?.technical?.pass) {
    if (!original?.technical?.schemaValid) {
      parts.push("Schema was invalid in original — now valid or invalid");
    } else {
      parts.push("Technical pass state changed");
    }
  }
  if (
    result.reasoningQuality.status !== original?.reasoningQuality?.status &&
    !(
      result.reasoningQuality.status === "not_evaluated" &&
      !original?.reasoningQuality?.status
    )
  ) {
    const oldStatus = original?.reasoningQuality?.status || "not_set";
    const newStatus = result.reasoningQuality.status;
    if (newStatus === "passed") {
      parts.push(
        "Reasoning status changed from " +
          oldStatus +
          " to passed — expectedBehaviours now cover the output",
      );
    } else if (oldStatus === "passed" || oldStatus === "not_set") {
      parts.push("Reasoning status degraded from " + oldStatus + " to failed");
    }
  }

  // Check for schema failure change
  if (result.technical.schemaValid !== original?.technical?.schemaValid) {
    if (!result.technical.schemaValid) {
      parts.push("Schema validation failure — reasoning marked not_evaluated");
    } else {
      parts.push("Schema validation restored");
    }
  }

  return parts.join("; ") || "No detailed change explanation available";
}

// ═══════════════════════════════════════════════════════
//  MAIN
// ═══════════════════════════════════════════════════════

async function main() {
  // ── SAVED MODE ──────────────────────────────────────
  if (mode === "saved") {
    const { summary, directory, timestamp } =
      await loadSavedResults(resultsDir);

    // Load test cases with expectedBehaviours from diagnostic data
    let testCases;
    const diagPath = join(__dirname, "data", "live-diagnostic-v0.2.json");
    if (existsSync(diagPath)) {
      testCases = JSON.parse(readFileSync(diagPath, "utf-8"));
      // Only include cases that exist in the saved results
      testCases = testCases.filter((tc) =>
        summary.testCaseResults.some((sr) => sr.id === tc.id),
      );
    }

    if (!testCases || !testCases.length) {
      console.log(`\n📊 Saved Results Re-evaluation`);
      console.log(`   Directory: ${directory}`);
      console.log(`   Cases loaded: ${summary.casesRun}\n`);
      console.log(
        "⚠️  No test cases with expectedBehaviours found. Cannot re-evaluate.\n",
      );
      return;
    }

    console.log(`\n📊 Saved Live Results Re-evaluation`);
    console.log(`   Source directory: ${directory}`);
    console.log(`   Cases to re-evaluate: ${testCases.length}`);
    console.log(`   Provider (from source): qwen-claude:latest / Ollama\n`);

    // Re-run scoring with new behaviour-based logic against saved analysis results
    const { results, providerMetadata } = await reEvaluateSavedLiveResults(
      directory,
      testCases,
    );

    // Compute aggregate metrics
    const total = results.length;
    const techPassCount = results.filter((r) => r.technical.pass).length;
    const techSchemaValidCount = results.filter(
      (r) => r.technical.schemaValid,
    ).length;
    const techClassMatchCount = results.filter(
      (r) => r.technical.classificationMatch,
    ).length;
    const techNqPresentCount = results.filter(
      (r) => r.technical.nextQuestionPresent,
    ).length;

    const rqStatuses = {};
    for (const r of results) {
      const s = r.reasoningQuality.status || "not_evaluated";
      rqStatuses[s] = (rqStatuses[s] || 0) + 1;
    }
    const combinedPassCount = results.filter(
      (r) =>
        r.technical.pass &&
        r.technical.schemaValid &&
        r.reasoningQuality.status === "passed",
    ).length;
    const coveredCases = results
      .map((r) => r.reasoningQuality.behaviourCoverage)
      .filter((c) => c.coverage !== "n/a");
    const avgBehaviourCoverage =
      coveredCases.length > 0
        ? coveredCases.reduce((s, c) => s + c.coverage, 0) / coveredCases.length
        : 0;

    // Determine which cases changed from original evaluation
    const originalMap = {};
    for (const sr of summary.testCaseResults || []) {
      originalMap[sr.id] = sr;
    }
    const changes = [];
    for (const r of results) {
      const orig = originalMap[r.id];
      if (!orig) continue;
      const techChanged = r.technical.pass !== orig.technical.pass;
      const rqStatusChanged =
        r.reasoningQuality.status !== orig.reasoningQuality?.status &&
        !(
          r.reasoningQuality.status === "not_evaluated" &&
          !orig.reasoningQuality?.status
        );
      if (techChanged || rqStatusChanged) {
        changes.push({
          id: r.id,
          oldTechnicalPass: orig.technical.pass,
          newTechnicalPass: r.technical.pass,
          oldReasoningStatus: orig.reasoningQuality?.status || "not_set",
          newReasoningStatus: r.reasoningQuality.status,
          reason: getChangeExplanation(r, orig),
        });
      }
    }

    // Save re-evaluated results with provenance
    const reEvalTimestamp = new Date()
      .toISOString()
      .replace(/[:.]/g, "-")
      .slice(0, 19);
    const saveDir = join(resultsDir, `re-eval-${timestamp}-${reEvalTimestamp}`);
    mkdirSync(saveDir, { recursive: true });

    for (const r of results) {
      const tc = testCases.find((t) => t.id === r.id);
      // Full result with provenance metadata
      writeFileSync(
        join(saveDir, `${r.id}-result.json`),
        JSON.stringify(
          {
            id: r.id,
            input: r.input,
            description: r.description,
            responseDurationMs: r.responseDurationMs,
            actualPrimaryType: r.actualPrimaryType,
            actualReasoningModes: r.actualReasoningModes,
            technical: r.technical,
            reasoningQuality: {
              status: r.reasoningQuality.status,
              classificationAcceptanceNotes:
                r.reasoningQuality.classificationAcceptanceNotes || [],
              normalisationsApplied:
                r.reasoningQuality.normalisationsApplied || [],
              behaviourCoverage: r.reasoningQuality.behaviourCoverage,
              behaviours: r.reasoningQuality.behaviours,
              requiredConcepts: {
                // renamed to indicate diagnostic-only role
                pass: r.reasoningQuality.requiredConcepts.pass,
                details: r.reasoningQuality.requiredConcepts.details,
                _note: "diagnostic compatibility metric — not authoritative",
              },
              unsupportedInferencesAbsent:
                r.reasoningQuality.unsupportedInferencesAbsent,
              pass: r.reasoningQuality.pass,
            },
            provenance: {
              sourceRunDirectory: directory,
              sourceProvider: "qwen-claude:latest",
              sourceModel: "qwen-claude:latest",
              modelWasCalled: false,
              evaluationTimestamp: new Date().toISOString(),
              evaluatorVersion: "0.2-behaviour-authoritative",
            },
            originalRawOutputSnippet: r._originalRawOutput
              ? r._originalRawOutput.slice(0, 500)
              : null,
          },
          null,
          2,
        ),
      );

      if (tc) {
        writeFileSync(
          join(saveDir, `${r.id}-summary.md`),
          generateMarkdownReport(r, tc),
        );
      }
    }

    // Save re-evaluated summary
    const reEvalSummary = {
      timestamp: new Date().toISOString(),
      provider: "qwen-claude:latest",
      promptVersion: "v0.2",
      casesRun: total,
      provenance: {
        sourceRunDirectory: directory,
        sourceProvider: "qwen-claude:latest",
        sourceModel: "qwen-claude:latest",
        modelWasCalled: false,
        evaluationTimestamp: new Date().toISOString(),
        evaluatorVersion: "0.2-behaviour-authoritative",
      },
      summary: {
        technical: {
          schemaValidityRate: `${((techSchemaValidCount / total) * 100).toFixed(1)}%`,
          classificationMatchRate: `${((techClassMatchCount / total) * 100).toFixed(1)}%`,
          nextQuestionPresentRate: `${((techNqPresentCount / total) * 100).toFixed(1)}%`,
          passRate: `${((techPassCount / total) * 100).toFixed(1)}%`,
        },
        reasoningQuality: {
          statusDistribution: rqStatuses,
          passRate: `${(((rqStatuses.passed || 0) / total) * 100).toFixed(1)}%`,
          averageBehaviourCoverage: `${(avgBehaviourCoverage * 100).toFixed(1)}%`,
        },
        combinedPassRate: `${((combinedPassCount / total) * 100).toFixed(1)}%`,
        averageResponseDurationMs: Math.round(
          results.reduce((s, r) => s + (r.responseDurationMs || 0), 0) / total,
        ),
      },
      testCaseResults: results.map((r) => ({
        id: r.id,
        input: r.input,
        description: r.description,
        responseDurationMs: r.responseDurationMs,
        actualPrimaryType: r.actualPrimaryType,
        actualReasoningModes: r.actualReasoningModes,
        technical: r.technical,
        reasoningQuality: {
          status: r.reasoningQuality.status,
          behaviourCoverage: r.reasoningQuality.behaviourCoverage,
          requiredConcepts: {
            pass: r.reasoningQuality.requiredConcepts.pass,
            details: r.reasoningQuality.requiredConcepts.details,
          },
          unsupportedInferencesAbsent:
            r.reasoningQuality.unsupportedInferencesAbsent,
          pass: r.reasoningQuality.pass,
        },
      })),
    };
    writeFileSync(
      join(saveDir, "summary.json"),
      JSON.stringify(reEvalSummary, null, 2),
    );

    // ── Console output ───────────────────────────────────
    console.log("\n" + "=".repeat(64));
    console.log("SAVED LIVE RE-EVALUATION SUMMARY");
    console.log(`${"=".repeat(64)}\n`);

    console.log(`Source:                       ${directory}\n`);

    const CYAN = "\x1b[36m",
      YELLOW = "\x1b[33m",
      GREEN = "\x1b[32m",
      RESET = "\x1b[0m";
    console.log(`${CYAN}─── TECHNICAL ──────────────────────────────${RESET}`);
    console.log(
      `  Schema validity rate:     ${GREEN}${techSchemaValidCount}/${total} ${((techSchemaValidCount / total) * 100).toFixed(1)}%${RESET}`,
    );
    console.log(
      `  Classification match:     ${techClassMatchCount}/${total} ${((techClassMatchCount / total) * 100).toFixed(1)}%`,
    );
    console.log(
      `  Next-question present:    ${techNqPresentCount}/${total} ${((techNqPresentCount / total) * 100).toFixed(1)}%`,
    );
    console.log(
      `  Technical pass rate:      ${techPassCount}/${total} ${((techPassCount / total) * 100).toFixed(1)}%\n`,
    );

    console.log(
      `${YELLOW}─── REASONING QUALITY ───────────────────────${RESET}`,
    );
    console.log(
      `  Status distribution:      ${Object.entries(rqStatuses)
        .map(([s, c]) => `${s}:${c}`)
        .join(", ")}`,
    );
    console.log(
      `  Reasoning pass rate:      ${rqStatuses.passed || 0}/${total} ${(((rqStatuses.passed || 0) / total) * 100).toFixed(1)}%\n`,
    );

    const coveredCases2 = results
      .map((r) => r.reasoningQuality.behaviourCoverage)
      .filter((c) => c.coverage !== "n/a");
    console.log(
      `${YELLOW}─── BEHAVIOUR COVERAGE ─────────────────────-${RESET}`,
    );
    if (coveredCases2.length) {
      console.log(
        `  Average coverage:         ${(avgBehaviourCoverage * 100).toFixed(1)}% across ${coveredCases2.length} cases\n`,
      );
    } else {
      console.log(`  No behavioural expectations defined.\n`);
    }

    console.log(`${"─".repeat(64)}`);
    console.log(
      `  Both technical + reasoning: ${GREEN}${combinedPassCount}/${total} ${((combinedPassCount / total) * 100).toFixed(1)}%${RESET}`,
    );
    if (changes.length) {
      console.log(`\nCases changed from original evaluation:`);
      for (const ch of changes) {
        console.log(
          `  ${ch.id}: tech ${oldStr(ch.oldTechnicalPass)} → ${newStr(ch.newTechnicalPass)}, ` +
            `reasoning ${oldStr(ch.oldReasoningStatus)} → ${newStr(ch.newReasoningStatus)}`,
        );
        if (ch.reason) console.log(`    Reason: ${ch.reason}`);
      }
    } else {
      console.log(`  No cases changed from original evaluation.`);
    }

    const schemaInvalidCases = results.filter((r) => !r.technical.schemaValid);
    if (schemaInvalidCases.length) {
      console.log(
        `\nSchema-invalid cases (not evaluated): ${schemaInvalidCases.map((r) => r.id).join(", ")}`,
      );
    }

    console.log(`${"=".repeat(64)}\n`);
    console.log(`Results saved to: ${saveDir}/`);
    return;
  }

  // ── DIAGNOSTIC / NORMAL MODE ────────────────────────
  let testCases;
  if (testDataPath) {
    testCases = loadTestCases(testDataPath);
    console.log(`\n⚡ Confidence Engine v0.2 — Semantic Reasoning Evaluator`);
    console.log(`   Provider: ${useRealProvider ? "Ollama (real)" : "Mock"}`);
    console.log(
      `   Mode:     ${mode === "diagnostic" ? "Live diagnostic" : "Standard"}`,
    );
    console.log(`   Cases loaded: ${testCases.length}\n`);
  }

  // ── Analysis function setup ─────────────────────────
  let analyseScenarioFn;
  if (useRealProvider) {
    const { analyseScenario } = await import("../lib/analysis.js");
    analyseScenarioFn = analyseScenario;
  } else {
    const mockProvider = new MockProvider();
    const schemaMod = await import("../lib/reconstruction/schema.js");
    const {
      reconstructionV2Schema,
      reconstructionSchema: reconstructionV1Schema,
    } = schemaMod;
    const { buildPrompt } = await import("../lib/reconstruction/prompt.js");

    analyseScenarioFn = async (scenario, opts = {}) => {
      const startTime = Date.now();
      if (!scenario?.trim())
        return {
          success: false,
          error: "Empty scenario",
          responseDurationMs: 0,
        };

      let promptObj;
      try {
        promptObj = await buildPrompt(
          scenario.trim(),
          opts.promptVersion || "v0.2",
        );
      } catch {
        promptObj = { prompt: scenario, version: "v0.2" };
      }

      const mockResult = await mockProvider.generateReconstruction(
        promptObj.prompt || scenario,
        "",
      );

      let schemaValid = false;
      let validatedData = null;
      if (reconstructionV2Schema?.safeParse) {
        const v2R = reconstructionV2Schema.safeParse(mockResult);
        if (v2R.success) {
          schemaValid = true;
          validatedData = v2R.data;
        } else {
          const v1R = reconstructionV1Schema?.safeParse(mockResult);
          if (v1R?.success) {
            schemaValid = true;
            validatedData = v1R.data;
          }
        }
      }

      if (!schemaValid || !validatedData) {
        return {
          success: false,
          validationStatus: "invalid",
          responseDurationMs: Date.now() - startTime,
        };
      }

      return {
        success: true,
        validationStatus: "valid",
        responseDurationMs: Date.now() - startTime,
        inputClassification: validatedData.inputClassification,
        reconstruction: validatedData.reconstruction,
        evidence: validatedData.evidence,
        nextQuestion: validatedData.nextQuestion,
      };
    };
  }

  // ── Run cases ───────────────────────────────────────
  let results = [];
  if (mode === "saved" && !testCases) {
    // Re-evaluate saved results from summary
    const { summary: savedSummary, directory } =
      await loadSavedResults(resultsDir);
    testCases = savedSummary.testCaseResults.map((r, i) => ({
      id: r.id,
      input: r.input,
      description: r.description || "",
      expectedPrimaryTypes: [],
      expectedReasoningModes: [],
      shouldIdentify: [],
      shouldNotInfer: [],
      expectedClassifications: [],
      expectedBehaviours: [],
    }));

    // Actually use the saved results directly — just re-format for output
    results = savedSummary.testCaseResults.map((r) => ({
      ...r,
      technical: { ...r.technical },
      reasoningQuality: { ...r.reasoningQuality },
    }));
  } else if (testCases) {
    for (const tc of testCases) {
      process.stdout.write(`  ${tc.id}: ... `);
      const r = await runTestCase(tc, analyseScenarioFn);
      results.push(r);
      const tStatus = r.technical.pass
        ? "\x1b[32m✅\x1b[0m"
        : "\x1b[31m❌\x1b[0m";
      const rqStatus =
        r.reasoningQuality.status === "passed"
          ? "\x1b[32m✅\x1b[0m"
          : r.reasoningQuality.status === "not_evaluated"
            ? "\x1b[33m⏭\x1b[0m"
            : "\x1b[31m❌\x1b[0m";
      process.stdout.write(`${tStatus} tech  ${rqStatus} reason\n`);

      if (!r.technical.pass && r.technical.errors?.length) {
        for (const e of r.technical.errors.slice(0, 2))
          process.stdout.write(`     → [tech] ${e}\n`);
      }
    }
  }

  // ── Summary stats ───────────────────────────────────
  const total = results.length;
  const techPassCount = results.filter((r) => r.technical.pass).length;
  const rqStatuses = {};
  for (const r of results) {
    const s = r.reasoningQuality.status || "not_evaluated";
    rqStatuses[s] = (rqStatuses[s] || 0) + 1;
  }
  const rqPassedCount = rqStatuses.passed || 0;
  const combinedPassCount = results.filter(
    (r) => r.technical.pass && r.reasoningQuality.pass,
  ).length;
  const avgDuration =
    total > 0
      ? results.reduce((s, r) => s + (r.responseDurationMs || 0), 0) / total
      : 0;

  // ── Console output ───────────────────────────────────
  console.log(`\n${"=".repeat(64)}`);
  console.log("EVALUATION SUMMARY");
  console.log(`${"=".repeat(64)}\n`);

  console.log(`Cases run:                  ${total}\n`);

  const CYAN = "\x1b[36m",
    YELLOW = "\x1b[33m",
    RESET = "\x1b[0m";
  console.log(`${CYAN}─── TECHNICAL ──────────────────────────────${RESET}`);
  const techSchemaValidCount = results.filter(
    (r) => r.technical.schemaValid,
  ).length;
  const techClassMatchCount = results.filter(
    (r) => r.technical.classificationMatch,
  ).length;
  const techNqCount = results.filter(
    (r) => r.technical.nextQuestionPresent,
  ).length;

  console.log(
    `  Schema validity rate:     ${techSchemaValidCount}/${total} ${((techSchemaValidCount / total) * 100).toFixed(1)}%`,
  );
  console.log(
    `  Classification match:     ${techClassMatchCount}/${total} ${((techClassMatchCount / total) * 100).toFixed(1)}%`,
  );
  console.log(
    `  Next-question present:    ${techNqCount}/${total} ${((techNqCount / total) * 100).toFixed(1)}%`,
  );
  console.log(
    `  Technical pass rate:      ${techPassCount}/${total} ${((techPassCount / total) * 100).toFixed(1)}%\n`,
  );

  console.log(`${YELLOW}─── REASONING QUALITY ───────────────────────${RESET}`);
  console.log(
    `  Status distribution:      ${Object.entries(rqStatuses)
      .map(([s, c]) => `${s}:${c}`)
      .join(", ")}`,
  );
  console.log(
    `  Reasoning pass rate:      ${rqPassedCount}/${total} ${((rqPassedCount / total) * 100).toFixed(1)}%\n`,
  );

  // Behaviour coverage aggregate
  const allCoverage = results.map((r) => r.reasoningQuality.behaviourCoverage);
  const covAvg =
    allCoverage
      .filter((c) => c.coverage !== "n/a")
      .reduce((s, c) => s + c.coverage, 0) /
    Math.max(allCoverage.filter((c) => c.coverage !== "n/a").length, 1);
  console.log(`${YELLOW}─── BEHAVIOUR COVERAGE ─────────────────────-${RESET}`);
  const coveredCases = allCoverage.filter((c) => c.coverage !== "n/a");
  if (coveredCases.length) {
    console.log(
      `  Average coverage:         ${(covAvg * 100).toFixed(1)}% across ${coveredCases.length} cases\n`,
    );
  } else {
    console.log(`  No behavioural expectations defined.\n`);
  }

  console.log(`${"─".repeat(64)}`);
  console.log(
    `  Both technical + reasoning: ${combinedPassCount}/${total} ${((combinedPassCount / total) * 100).toFixed(1)}%`,
  );
  if (techPassCount > combinedPassCount) {
    const onlyTech = results.filter(
      (r) => r.technical.pass && !r.reasoningQuality.pass,
    );
    console.log(
      `  Technical only:            ${onlyTech.length} — ${onlyTech.map((r) => r.id).join(", ")}`,
    );
  }
  if (rqPassedCount > combinedPassCount) {
    const onlyReason = results.filter(
      (r) => !r.technical.pass && r.reasoningQuality.status === "passed",
    );
    console.log(
      `  Reasoning only:            ${onlyReason.length} — ${onlyReason.map((r) => r.id).join(", ")}`,
    );
  }
  console.log(`  Avg response duration:    ${avgDuration.toFixed(0)}ms`);
  console.log(`${"=".repeat(64)}\n`);

  // ── Save results ────────────────────────────────
  const timestamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19);
  let saveDir;

  if (mode === "diagnostic") {
    saveDir = join(resultsDir, timestamp);
    mkdirSync(saveDir, { recursive: true });

    // Per-case JSON + Markdown
    for (const r of results) {
      const tc = testCases.find((t) => t.id === r.id);
      writeFileSync(
        join(saveDir, `${r.id}-result.json`),
        JSON.stringify(r, null, 2),
      );
      if (tc) {
        writeFileSync(
          join(saveDir, `${r.id}-summary.md`),
          generateMarkdownReport(r, tc),
        );
      }
    }

    const fullResults = generateFullSummaryJSON(
      results,
      testCases,
      useRealProvider ? "ollama-real" : "mock",
    );
    writeFileSync(
      join(saveDir, "summary.json"),
      JSON.stringify(fullResults, null, 2),
    );
    console.log(`Live diagnostic results saved to: ${saveDir}/`);

    const manifestPath = join(resultsDir, "latest-manifest.json");
    writeFileSync(
      manifestPath,
      JSON.stringify({ latestRun: timestamp, caseCount: total }, null, 2),
    );
    console.log(`Manifest saved to: ${manifestPath}`);
  } else {
    saveDir = resultsDir;
    const fullResults = generateFullSummaryJSON(
      results,
      testCases,
      useRealProvider ? "ollama-real" : "mock",
    );
    writeFileSync(
      join(saveDir, `evaluation-${timestamp}.json`),
      JSON.stringify(fullResults, null, 2),
    );
    console.log(
      `Results saved to: ${join(saveDir, `evaluation-${timestamp}.json`)}`,
    );
  }
}

// ── Helper to load test cases ────────────────────────
function loadTestCases(path) {
  const content = readFileSync(path, "utf-8");
  if (path.endsWith(".json")) return JSON.parse(content);
  return content
    .split("\n")
    .filter((l) => l.trim())
    .map((l) => JSON.parse(l));
}

if (process.env.EVAL_SKIP_MAIN) {
  // When imported by tests, skip the main() execution.
  // Pure functions are exported for direct testing.
} else {
  main().catch((e) => {
    console.error("Evaluator failed:", e.message);
    process.exit(1);
  });
}

// ── Exports for testing ─────────────────────────────
export {
  normalise,
  matchesAnyPhrase,
  matchesReasoningMode,
  matchesClassification,
  matchesSecondaryClassification,
  matchesStructuredField,
  checksImportantUnknowns,
  checksNextQuestionTarget,
  evaluateBehaviour,
  calculateBehaviourCoverage,
  normaliseEvidence,
  VALID_EVIDENCE_TYPES,
  EVIDENCE_ALIASES,
  generateMarkdownReport,
  generateFullSummaryJSON,
};