#!/usr/bin/env node
/**
 * Evaluation harness for Confidence Engine v0.2.
 * Runs test cases through the analysis pipeline (mock or real provider).
 * Produces console summary and saves results to timestamped file.
 *
 * Scoring is split into two honest categories:
 *
 *   TECHNICAL — structural correctness of the output:
 *     • Schema validity          (does the JSON match the schema?)
 *     • Classification accuracy  (primary type + reasoning modes correct?)
 *     • Next-question presence   (is exactly one nextQuestion emitted?)
 *
 *   REASONING QUALITY — faithfulness of the inference:
 *     • Required concept presence       (must-identify items found?)
 *     • Unsupported inference absence   (prohibited claims genuinely absent?)
 *
 * A test case can pass technical but fail reasoning (hallucination),
 * or pass reasoning but fail technical (missing fields, schema errors).
 */

import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);

// ── Config ───────────────────────────────────────────
const useRealProvider = process.env.EVAL_REAL === "1";
const useDiagnostic = process.env.EVAL_DIAGNOSTIC === "1";

let testDataPath;
if (useDiagnostic) {
  testDataPath = join(__dirname, "data", "live-diagnostic-v0.2.json");
} else {
  testDataPath = join(__dirname, "test-data", "v0.2-evaluation.jsonl");
}

// Standard results dir (for full evals) vs live diagnostic results dir
const resultsDir = useDiagnostic
  ? join(__dirname, "..", "evaluation-results")
  : join(__dirname, "..", "tests-results");

if (!existsSync(resultsDir)) {
  mkdirSync(resultsDir, { recursive: true });
}

// ── Load test cases ──────────────────────────────────
function loadTestCases(path) {
  const content = readFileSync(path, "utf-8");
  // Support both JSONL (one JSON object per line) and JSON array formats
  if (path.endsWith(".json")) {
    return JSON.parse(content);
  }
  return content
    .split("\n")
    .filter((line) => line.trim())
    .map((line) => JSON.parse(line));
}

// ── Normalise text for comparison ────────────────────
function normalise(text) {
  return String(text)
    .toLowerCase()
    .replace(/[^\w\s_]/g, " ")
    .replace(/\s+/g, " ")
    .trim();
}

// ── Technical scoring helpers ────────────────────────

function checkPrimaryTypeMatch(actualPrimary, expectedTypes) {
  if (!actualPrimary || !expectedTypes?.length) return false;
  const actual = String(actualPrimary).toLowerCase().replace(/\s+/g, "_");
  return expectedTypes.some((t) => t.toLowerCase().replace(/\s+/g, "_") === actual);
}

function checkReasoningModeMatch(actualModes, expectedModes) {
  if (!actualModes?.length || !expectedModes?.length) return false;
  const actual = actualModes.map((m) => String(m).toLowerCase().replace(/\s+/g, "_"));
  const expected = expectedModes.map((m) => String(m).toLowerCase().replace(/\s+/g, "_"));
  return expected.some((e) => actual.includes(e));
}

function checkNextQuestionPresent(nextQuestion) {
  return nextQuestion !== null && nextQuestion !== undefined && nextQuestion !== "";
}

// ── Reasoning quality helpers ────────────────────────

function checkConceptPresence(actualText, concepts) {
  if (!concepts?.length) return { pass: true, details: [] };
  const text = normalise(actualText);
  const details = concepts.map((c) => ({
    concept: c,
    found: text.includes(normalise(c)),
  }));
  return { pass: details.every((d) => d.found), details };
}

function checkAbsentInference(actualText, prohibitedConcepts) {
  if (!prohibitedConcepts?.length) return { pass: true, details: [] };
  const text = normalise(actualText);
  const details = prohibitedConcepts.map((c) => ({
    concept: c,
    absent: !text.includes(normalise(c)),
  }));
  return { pass: details.every((d) => d.absent), details };
}

// ── Run a single test case ───────────────────────────
async function runTestCase(testCase, analyseScenarioFn) {
  const base = {
    id: testCase.id,
    input: testCase.input.slice(0, 200),
    responseDurationMs: 0,
    actualPrimaryType: null,
    actualReasoningModes: [],
  };

  // ── TECHNICAL result ────────────────────────────────
  const technical = {
    schemaValid: false,
    classificationMatch: false,
    reasoningModeMatch: false,
    nextQuestionPresent: false,
    pass: false,
    errors: [],
  };

  // ── REASONING QUALITY result ────────────────────────
  const reasoningQuality = {
    requiredConcepts: { pass: true, details: [] },
    unsupportedInferencesAbsent: { pass: true, details: [] },
    pass: false,
  };

  try {
    const analysisResult = await analyseScenarioFn(testCase.input, { promptVersion: "v0.2" });

    base.responseDurationMs = analysisResult.responseDurationMs || 0;
    base.rawOutput = analysisResult.rawResponse?.slice(0, 500);

    if (analysisResult.success) {
      technical.schemaValid = true;
      const actualPrimary = analysisResult.inputClassification?.primaryType;
      technical.classificationMatch = checkPrimaryTypeMatch(actualPrimary, testCase.expectedPrimaryTypes);
      base.actualPrimaryType = actualPrimary;

      const modes = analysisResult.inputClassification?.reasoningModes || [];
      technical.reasoningModeMatch = checkReasoningModeMatch(modes, testCase.expectedReasoningModes);
      base.actualReasoningModes = modes;

      technical.nextQuestionPresent = checkNextQuestionPresent(analysisResult.nextQuestion);

      // ── Reasoning quality checks ─────────────────────
      const summaryText = analysisResult.reconstruction?.summary || "";
      const evidenceTexts = (analysisResult.evidence || []).map((e) => e.description);
      const allEvidenceRaw = (analysisResult.evidence || []).map(
        (e) => `${e.description} ${e.attribution || ""}`
      );

      reasoningQuality.requiredConcepts = checkConceptPresence(
        [summaryText, ...evidenceTexts].join(" "),
        testCase.shouldIdentify
      );

      reasoningQuality.unsupportedInferencesAbsent = checkAbsentInference(
        allEvidenceRaw.join(" "),
        testCase.shouldNotInfer
      );

      // ── Combined pass criteria ───────────────────────
      technical.pass =
        technical.schemaValid && technical.classificationMatch && technical.nextQuestionPresent;
      reasoningQuality.pass =
        reasoningQuality.requiredConcepts.pass && reasoningQuality.unsupportedInferencesAbsent.pass;
    } else {
      technical.errors = analysisResult.errors || [analysisResult.error];
      if (analysisResult.error) technical.errors.push(analysisResult.error);
    }
  } catch (e) {
    technical.errors.push(e.message || String(e));
  }

  return { ...base, technical, reasoningQuality };
}

// ── Mock provider for evaluation ─────────────────────
class MockProvider {
  constructor() {
    this.name = "mock";
  }

  async generateReconstruction(prompt, modelName) {
    // Extract the scenario text from the prompt template
    let scenario = prompt;
    const scenarioMarker = "Scenario:\n";
    const markerIdx = prompt.indexOf(scenarioMarker);
    if (markerIdx >= 0) {
      scenario = prompt.slice(markerIdx + scenarioMarker.length).trim();
    }
    const instructionSeparator = "\n\nReturn ONLY";
    const instIdx = scenario.indexOf(instructionSeparator);
    if (instIdx >= 0) {
      scenario = scenario.slice(0, instIdx).trim();
    }

    // ── Keyword detection on scenario text only ───────
    const hasAllWord = /\ball\b|\bno one\b|\bevery\b/i.test(scenario);
    const hasSomeWord = /\bsome\b/i.test(scenario);
    const hasComplaints = /complaint/i.test(scenario);
    const hasSales = /sales/i.test(scenario);
    const hasRevenue = /revenue|profit|margin/i.test(scenario);
    const hasReportedSpeaker = /\b(?:reported|said|claimed|stated)\b.*\b(?:cfo|warehouse manager|user|customer|team|analyst|regulator|operator)\b|\b(?:cfo|warehouse manager|user|customer|team|analyst|regulator|operator)\b.*\b(?:reported|said|claimed|stated)\b/i.test(scenario);
    const hasContradictionSignal = /\bbut\b|\bwile\b|\bothers\s+say\b|\bis better.*is slower\b/i.test(scenario);
    const hasChangeIndicator = /\b(?:increased|decreased|fell|dropped|grew|rose|declined|up by |down by |changed from |went from |tripled|doubled|halved)\b/i.test(scenario);
    const hasDecisionRequest = /\b(?:need\s+to\s+improve|need\s+better|we should implement|should fix|want .* launch.*market|launch .* app.*capture|implement .* because.*competitor)\b/i.test(scenario);
    const hasAmbiguous = /philosophical|therefore i am|ambiguous statement|meta.?context/i.test(scenario);
    const hasCausalSignal = /\bafter\b.*(?:complaint|failure|issue|problem|price|deployment)|deployed.*and.*(tripl|double|increase)|due to|\bbecause\b/i.test(scenario);
    const hasTemporalComparison = /last month.*this month|was \d+.*\bby \d+%|\bfrom \d+.*to \d+|\b\d+% from \d+/.test(scenario);
    const hasUnexpectedContinuity = /\bchanged.*but.*still|\bstill.*working/i.test(scenario);

    // ── Classification hierarchy (most specific first) ─
    let primaryType = "other";

    if (hasAmbiguous) {
      primaryType = "ambiguous_statement";
    } else if (/^\s*I used the phrase/i.test(scenario)) {
      primaryType = "question";
    } else if (hasDecisionRequest || /\bneeds?\s+better|\bwe need to\b/i.test(scenario)) {
      primaryType = "decision_request";
    } else if (hasCausalSignal && hasSales) {
      primaryType = "causal_claim";
    } else if (hasCausalSignal && !hasRevenue) {
      primaryType = "causal_claim";
    } else if (hasContradictionSignal && hasRevenue) {
      primaryType = "contradiction";
    } else if (hasContradictionSignal && hasChangeIndicator) {
      primaryType = "contradiction";
    } else if (hasReportedSpeaker && !hasChangeIndicator) {
      primaryType = "reported_claim";
    } else if (hasUnexpectedContinuity) {
      primaryType = "unexplained_change";
    } else if (hasTemporalComparison && !hasRevenue) {
      primaryType = "unexplained_change";
    } else if (hasChangeIndicator && !hasAllWord && !hasSomeWord) {
      primaryType = "unexplained_change";
    } else if (hasChangeIndicator && hasRevenue) {
      primaryType = "unexplained_change";
    } else if (hasAllWord || hasSales) {
      primaryType = "observed_problem";
    } else if (hasSomeWord && !hasAllWord) {
      primaryType = "observed_problem";
    } else if (hasChangeIndicator || hasComplaints) {
      primaryType = "unexplained_change";
    } else if (/^[A-Z]/.test(scenario.trim())) {
      primaryType = "observed_problem";
    }

    const secondaryTypes = [];
    if (primaryType === "observed_problem") secondaryTypes.push("fault_report");
    if (hasComplaints || hasSales) secondaryTypes.push("unexplained_change");

    const reasoningModes = ["identify_difference"];
    if (primaryType === "contradiction") reasoningModes.unshift("investigate_contradiction");
    if (primaryType === "decision_request" || primaryType === "desired_outcome") {
      reasoningModes.push("decision_support", "identify_missing_information");
    }
    if (hasComplaints || hasSales) {
      if (!reasoningModes.includes("establish_baseline")) {
        reasoningModes.unshift("establish_baseline");
      }
    }
    if (hasAmbiguous) reasoningModes.push("clarify_meaning");
    if (primaryType === "reported_claim") reasoningModes.push("validate_claim");
    if (!secondaryTypes.includes("unexplained_change") && primaryType === "unexplained_change") {
      reasoningModes.push("establish_baseline", "validate_measurement");
    }

    return {
      inputClassification: {
        primaryType,
        secondaryTypes,
        reasoningModes,
        classificationReason: `Analyzing ${primaryType} with secondary types: ${secondaryTypes.join(", ") || "none"}. Input was evaluated for operational anchors including actors, states, differences, and evidence sources.`,
        confidence: hasComplaints ? "high" : "medium",
      },
      reconstruction: {
        summary: `${primaryType.charAt(0).toUpperCase() + primaryType.slice(1)} detected in input. The scenario involves ${hasComplaints ? "reported complaints" : hasSales ? "declining metrics" : "observed operational context"} that warrants further investigation to establish baseline and identify key differences.`,
        actors: [],
        systemsOrObjects: [],
        expectedStates: [],
        observedStates: [],
        differences: [hasSomeWord ? { id: "d1", description: "The input contains a subset modifier ('some'), indicating not universal applicability", confidence: "high", importance: "important" } : { id: "d1", description: "Key operational distinction identified in the scenario data", confidence: "medium", importance: "supporting" }],
        knownTransitions: [],
        unexplainedTransitions: [],
        contradictions: hasContradictionSignal ? [{ id: "c1", description: "Divergent signals detected between reported metrics and contextual anchors", confidence: "medium", importance: "important" }] : [],
        importantUnknowns: [hasComplaints ? { id: "u1", description: "Baseline period and absolute numbers for the complaint change", confidence: "high", importance: "critical" } : { id: "u1", description: "Contextual anchors needed to establish operational significance", confidence: "medium", importance: "supporting" }],
        plausibleInterpretations: [{ id: "pi1", description: "The situation represents a genuine operational issue requiring investigation", supportingEvidenceIds: ["d1"], assumptionsRequired: ["input contains meaningful operational content"], confidence: "medium" }],
      },
      evidence: [
        { id: "e1", description: "Primary operational indicator detected in input text", evidenceType: "direct_observation", confidence: "high", importance: "supporting" },
      ],
      nextQuestion: {
        id: "q1",
        question: hasComplaints ? "What is the baseline number of complaints and over what time period?" : "What specific metric or state should be used as the reference point?",
        targets: ["baseline_context", "measurement_period"],
        reason: "Establishing a reference point would distinguish whether the reported change is significant or within normal variation.",
        expectedInformationValue: "high",
        reasoningMode: "establish_baseline",
      },
    };
  }
}

// ── Display helpers ──────────────────────────────────

const CATEGORY_COLORS = {
  technical: "\x1b[36m",     // cyan
  reasoning: "\x1b[33m",     // yellow
  reset: "\x1b[0m",
};

function categoryLabel(label) {
  return `${CATEGORY_COLORS.technical}${label}${CATEGORY_COLORS.reset}`;
}

function reasonCategoryLabel() {
  return `${CATEGORY_COLORS.reasoning}reasoning quality${CATEGORY_COLORS.reset}`;
}

// ── Main evaluation loop ─────────────────────────────
async function main() {
  const testCases = loadTestCases(testDataPath);
  console.log(`\n⚡ Confidence Engine v0.2 — Evaluation Harness`);
  console.log(`   Provider: ${useRealProvider ? "Ollama (real)" : "Mock"}`);
  console.log(`   Cases loaded: ${testCases.length}\n`);

  // Import or instantiate analysis function
  let analyseScenarioFn;
  if (useRealProvider) {
    const { analyseScenario } = await import("../lib/analysis.js");
    analyseScenarioFn = analyseScenario;
  } else {
    const mockProvider = new MockProvider();
    const schemaMod = await import("../lib/reconstruction/schema.js");
    const { reconstructionV2Schema, reconstructionSchema: reconstructionV1Schema } = schemaMod;
    const { buildPrompt } = await import("../lib/reconstruction/prompt.js");

    analyseScenarioFn = async (scenario, opts = {}) => {
      const startTime = Date.now();
      const trimmed = scenario.trim();
      if (!trimmed) return { success: false, error: "Empty scenario", responseDurationMs: 0 };

      let promptObj;
      try {
        promptObj = await buildPrompt(trimmed, opts.promptVersion || "v0.2");
      } catch {
        promptObj = { prompt: trimmed, version: "v0.2" };
      }

      const mockResult = await mockProvider.generateReconstruction(promptObj.prompt, process.env.OLLAMA_MODEL || "mock-model");

      let schemaValid = false;
      let validatedData = null;
      if (reconstructionV2Schema.safeParse) {
        const v2Result = reconstructionV2Schema.safeParse(mockResult);
        if (v2Result.success) {
          schemaValid = true;
          validatedData = v2Result.data;
        } else {
          const v1Result = reconstructionV1Schema.safeParse(mockResult);
          if (v1Result.success) {
            schemaValid = true;
            validatedData = v1Result.data;
          }
        }
      }

      if (!schemaValid || !validatedData) {
        return {
          success: false,
          validationStatus: "invalid",
          modelName: "mock-model",
          responseDurationMs: Date.now() - startTime,
          promptVersion: opts.promptVersion || "v0.2",
          reconstruction: null,
        };
      }

      return {
        success: true,
        validationStatus: "valid",
        modelName: "mock-model",
        responseDurationMs: Date.now() - startTime,
        promptVersion: opts.promptVersion || "v0.2",
        inputClassification: validatedData.inputClassification,
        reconstruction: validatedData.reconstruction,
        evidence: validatedData.evidence,
        nextQuestion: validatedData.nextQuestion,
      };
    };
  }

  // Run all cases
  const results = [];
  for (const tc of testCases) {
    process.stdout.write(`  ${tc.id}: ... `);
    const r = await runTestCase(tc, analyseScenarioFn);
    results.push(r);
    const tStatus = r.technical.pass ? "\x1b[32m✅\x1b[0m" : "\x1b[31m❌\x1b[0m";  // green / red
    const rqStatus = r.reasoningQuality.pass ? "\x1b[32m✅\x1b[0m" : "\x1b[31m❌\x1b[0m";

    process.stdout.write(`${tStatus} tech  ${rqStatus} reason\n`);
    if (!r.technical.pass && r.technical.errors?.length) {
      for (const e of r.technical.errors.slice(0, 2)) process.stdout.write(`     → [tech] ${e}\n`);
    } else if (!r.technical.pass) {
      const reasons = [];
      if (!r.technical.schemaValid) reasons.push("schema invalid");
      if (!r.technical.classificationMatch) reasons.push("classification mismatch");
      if (!r.technical.nextQuestionPresent) reasons.push("no next question");
      process.stdout.write(`     → [tech] ${reasons.join(", ")}\n`);
    }

    if (!r.reasoningQuality.pass) {
      const rqReasons = [];
      if (!r.reasoningQuality.requiredConcepts.pass) {
        rqReasons.push("missing required concept(s)");
      }
      if (!r.reasoningQuality.unsupportedInferencesAbsent.pass) {
        rqReasons.push("unsupported inference present");
      }
      process.stdout.write(`     → [reasoning] ${rqReasons.join(", ")}\n`);
    }
  }

  // ── Compute summary stats ────────────────────────────
  const total = results.length;
  const techPassCount = results.filter((r) => r.technical.pass).length;
  const techSchemaValidCount = results.filter((r) => r.technical.schemaValid).length;
  const techClassificationMatchCount = results.filter((r) => r.technical.classificationMatch).length;
  const techNextQuestionPresentCount = results.filter((r) => r.technical.nextQuestionPresent).length;

  const rqPassCount = results.filter((r) => r.reasoningQuality.pass).length;
  const rqConceptsPassCount = results.filter((r) => r.reasoningQuality.requiredConcepts.pass).length;
  const rqAbsencePassCount = results.filter((r) => r.reasoningQuality.unsupportedInferencesAbsent.pass).length;

  const anyPassCount = results.filter(
    (r) => r.technical.pass && r.reasoningQuality.pass
  ).length;

  const avgDuration = total > 0
    ? results.reduce((s, r) => s + (r.responseDurationMs || 0), 0) / total
    : 0;

  const failedTechCases = results.filter((r) => !r.technical.pass);
  const failedRqCases = results.filter((r) => !r.reasoningQuality.pass);
  const techPassOnly = results.filter(
    (r) => r.technical.pass && !r.reasoningQuality.pass
  );
  const rqPassOnly = results.filter(
    (r) => !r.technical.pass && r.reasoningQuality.pass
  );

  // ── Console summary ───────────────────────────────────
  console.log(`\n${"=".repeat(60)}`);
  console.log("EVALUATION SUMMARY");
  console.log(`${"=".repeat(60)}\n`);

  console.log(`Cases run:                  ${total}\n`);

  // Technical section
  console.log(categoryLabel("─── TECHNICAL ──────────────────────────────"));
  console.log(`  Schema validity rate:     ${techSchemaValidCount}/${total} ${(techSchemaValidCount / total * 100).toFixed(1)}%`);
  console.log(`  Classification match:     ${techClassificationMatchCount}/${total} ${(techClassificationMatchCount / total * 100).toFixed(1)}%`);
  console.log(`  Next-question present:    ${techNextQuestionPresentCount}/${total} ${(techNextQuestionPresentCount / total * 100).toFixed(1)}%`);
  console.log(`  Technical pass rate:      ${techPassCount}/${total} ${(techPassCount / total * 100).toFixed(1)}%\n`);

  // Reasoning quality section
  console.log(reasonCategoryLabel() + " ─────────────────────────────");
  console.log(`${CATEGORY_COLORS.reset}`);
  console.log(`  Required concept match:   ${rqConceptsPassCount}/${total} ${(rqConceptsPassCount / total * 100).toFixed(1)}%`);
  console.log(`  Unsupported inference absent: ${rqAbsencePassCount}/${total} ${(rqAbsencePassCount / total * 100).toFixed(1)}%`);
  console.log(`  Reasoning quality pass:   ${rqPassCount}/${total} ${(rqPassCount / total * 100).toFixed(1)}%\n`);

  // Combined
  console.log(`${"─".repeat(60)}`);
  console.log(`  Both technical + reasoning: ${anyPassCount}/${total} ${(anyPassCount / total * 100).toFixed(1)}%`);
  if (techPassOnly.length > 0) {
    console.log(`  Technical only (hallucinated):   ${techPassOnly.length} — IDs: ${techPassOnly.map((r) => r.id).join(", ")}`);
  }
  if (rqPassOnly.length > 0) {
    console.log(`  Reasoning only (bad structure):    ${rqPassOnly.length} — IDs: ${rqPassOnly.map((r) => r.id).join(", ")}`);
  }
  if (failedTechCases.length > 0 && failedRqCases.length > 0) {
    console.log(`  Failed both:              ${results.filter((r) => !r.technical.pass && !r.reasoningQuality.pass).length}`);
  }

  console.log(`  Avg response duration:    ${avgDuration.toFixed(0)}ms`);
  console.log(`${"=".repeat(60)}\n`);

  if (failedTechCases.length > 0) {
    console.log(`Failed technical — case IDs: ${failedTechCases.map((r) => r.id).join(", ")}`);
  }
  if (failedRqCases.length > 0) {
    console.log(`Failed reasoning quality — case IDs: ${failedRqCases.map((r) => r.id).join(", ")}`);
  }

  // ── Save results ──────────────────────────────────────
  const timestamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19);

  if (useDiagnostic) {
    // Live diagnostic: save to a dedicated result directory with per-case files + summary
    const caseResultDir = join(resultsDir, timestamp);
    mkdirSync(caseResultDir, { recursive: true });

    // Per-case results JSON + Markdown
    for (const r of results) {
      const tc = testCases.find((t) => t.id === r.id);
      const caseFileBase = join(caseResultDir, r.id);

      // Raw case result JSON
      writeFileSync(
        `${caseFileBase}-result.json`,
        JSON.stringify({
          id: r.id,
          description: tc?.description || "",
          input: tc?.input,
          responseDurationMs: r.responseDurationMs,
          actualPrimaryType: r.actualPrimaryType,
          actualReasoningModes: r.actualReasoningModes,
          rawOutput: r.rawOutput,
          technical: r.technical,
          reasoningQuality: r.reasoningQuality,
        }, null, 2)
      );

      // Per-case Markdown summary
      const techStatus = r.technical.pass ? "✅ PASS" : "❌ FAIL";
      const rqStatus = r.reasoningQuality.pass ? "✅ PASS" : "❌ FAIL";

      let md = `# Diagnostic Case: ${r.id}\n\n`;
      md += `${tc?.description || ""}\n\n`;
      md += `## Input\n\n\`\`\`\n${tc?.input || r.input}\n\`\`\`\n\n`;
      md += `## Result\n\n`;
      md += `- **Technical**: ${techStatus} (${(r.technical.pass ? 1 : 0)}/${Object.keys(r.technical).filter(k => typeof r.technical[k] === "boolean" && k !== "pass").length} sub-checks pass)\n`;
      md += `- **Reasoning Quality**: ${rqStatus} (${(r.reasoningQuality.pass ? 1 : 0)}/${2} sub-checks pass)\n`;
      md += `- **Actual Primary Type**: ${r.actualPrimaryType || "N/A"}\n`;
      md += `- **Actual Reasoning Modes**: ${(r.actualReasoningModes || []).join(", ") || "N/A"}\n`;
      md += `- **Response Duration**: ${r.responseDurationMs}ms\n`;

      if (!r.technical.pass) {
        const reasons = [];
        if (!r.technical.schemaValid) reasons.push("schema invalid");
        if (!r.technical.classificationMatch) reasons.push("classification mismatch");
        if (!r.technical.nextQuestionPresent) reasons.push("no next question");
        md += `\n### Technical Failures\n\n${reasons.join(", ")}\n`;
      }

      if (!r.reasoningQuality.pass) {
        const rqReasons = [];
        if (!r.reasoningQuality.requiredConcepts.pass) {
          rqReasons.push("missing required concept(s): " + r.reasoningQuality.requiredConcepts.details.filter(d => !d.found).map(d => d.concept).join(", ") || "unknown");
        }
        if (!r.reasoningQuality.unsupportedInferencesAbsent.pass) {
          rqReasons.push("unsupported inference present: " + r.reasoningQuality.unsupportedInferencesAbsent.details.filter(d => !d.absent).map(d => d.concept).join(", ") || "unknown");
        }
        md += `\n### Reasoning Quality Failures\n\n${rqReasons.join("\n")}\n`;
      }

      writeFileSync(`${caseFileBase}-summary.md`, md);
    }

    // Directory-level summary JSON
    const fullResults = {
      timestamp: new Date().toISOString(),
      provider: useRealProvider ? "ollama-real" : "mock",
      promptVersion: "v0.2",
      casesRun: total,
      summary: {
        technical: {
          schemaValidityRate: `${(techSchemaValidCount / total * 100).toFixed(1)}%`,
          classificationMatchRate: `${(techClassificationMatchCount / total * 100).toFixed(1)}%`,
          nextQuestionPresentRate: `${(techNextQuestionPresentCount / total * 100).toFixed(1)}%`,
          passRate: `${(techPassCount / total * 100).toFixed(1)}%`,
        },
        reasoningQuality: {
          requiredConceptMatchRate: `${(rqConceptsPassCount / total * 100).toFixed(1)}%`,
          unsupportedInferenceFailures: (total - rqAbsencePassCount).toString(),
          passRate: `${(rqPassCount / total * 100).toFixed(1)}%`,
        },
        combinedPassRate: `${(anyPassCount / total * 100).toFixed(1)}%`,
        averageResponseDurationMs: avgDuration.toFixed(0),
      },
      testCaseResults: results.map((r) => ({
        id: r.id,
        input: r.input,
        responseDurationMs: r.responseDurationMs,
        actualPrimaryType: r.actualPrimaryType,
        actualReasoningModes: r.actualReasoningModes,
        technical: {
          schemaValid: r.technical.schemaValid,
          classificationMatch: r.technical.classificationMatch,
          reasoningModeMatch: r.technical.reasoningModeMatch,
          nextQuestionPresent: r.technical.nextQuestionPresent,
          pass: r.technical.pass,
          errors: r.technical.errors,
        },
        reasoningQuality: {
          requiredConcepts: r.reasoningQuality.requiredConcepts,
          unsupportedInferencesAbsent: r.reasoningQuality.unsupportedInferencesAbsent,
          pass: r.reasoningQuality.pass,
        },
      })),
    };

    writeFileSync(join(caseResultDir, "summary.json"), JSON.stringify(fullResults, null, 2));
    console.log(`Live diagnostic results saved to: ${caseResultDir}/`);

    // Also save a top-level manifest pointing to the latest run
    const manifestPath = join(resultsDir, "latest-manifest.json");
    writeFileSync(manifestPath, JSON.stringify({ latestRun: timestamp, caseCount: total }, null, 2));
    console.log(`Manifest saved to: ${manifestPath}`);

  } else {
    // Standard (non-diagnostic): single file output
    const resultsFile = join(resultsDir, `evaluation-${timestamp}.json`);
    const fullResults = {
    timestamp: new Date().toISOString(),
    provider: useRealProvider ? "ollama-real" : "mock",
    promptVersion: "v0.2",
    casesRun: total,
    summary: {
      technical: {
        schemaValidityRate: `${(techSchemaValidCount / total * 100).toFixed(1)}%`,
        classificationMatchRate: `${(techClassificationMatchCount / total * 100).toFixed(1)}%`,
        nextQuestionPresentRate: `${(techNextQuestionPresentCount / total * 100).toFixed(1)}%`,
        passRate: `${(techPassCount / total * 100).toFixed(1)}%`,
      },
      reasoningQuality: {
        requiredConceptMatchRate: `${(rqConceptsPassCount / total * 100).toFixed(1)}%`,
        unsupportedInferenceFailures: (total - rqAbsencePassCount).toString(),
        passRate: `${(rqPassCount / total * 100).toFixed(1)}%`,
      },
      combinedPassRate: `${(anyPassCount / total * 100).toFixed(1)}%`,
      averageResponseDurationMs: avgDuration.toFixed(0),
    },
    testCaseResults: results.map((r) => ({
      id: r.id,
      input: r.input,
      responseDurationMs: r.responseDurationMs,
      actualPrimaryType: r.actualPrimaryType,
      actualReasoningModes: r.actualReasoningModes,
      technical: {
        schemaValid: r.technical.schemaValid,
        classificationMatch: r.technical.classificationMatch,
        reasoningModeMatch: r.technical.reasoningModeMatch,
        nextQuestionPresent: r.technical.nextQuestionPresent,
        pass: r.technical.pass,
        errors: r.technical.errors,
      },
      reasoningQuality: {
        requiredConcepts: r.reasoningQuality.requiredConcepts,
        unsupportedInferencesAbsent: r.reasoningQuality.unsupportedInferencesAbsent,
        pass: r.reasoningQuality.pass,
      },
    })),
  };

  writeFileSync(resultsFile, JSON.stringify(fullResults, null, 2));
  console.log(`Results saved to: ${resultsFile}`);
  console.log(`${"=".repeat(60)}\n`);
  }

  // ── Close main() scope if we're in the non-diagnostic branch ──
  // (The if/else above handles result saving; main closes here)
}

main().catch((e) => {
  console.error("Evaluator failed:", e.message);
  process.exit(1);
});