685 lines
31 KiB
JavaScript
Executable File
685 lines
31 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
|
/**
|
|
* Evaluation harness for Confidence Engine v0.2.
|
|
* Runs test cases through the analysis pipeline (mock or real provider).
|
|
* Produces console summary and saves results to timestamped file.
|
|
*
|
|
* Scoring is split into two honest categories:
|
|
*
|
|
* TECHNICAL — structural correctness of the output:
|
|
* • Schema validity (does the JSON match the schema?)
|
|
* • Classification accuracy (primary type + reasoning modes correct?)
|
|
* • Next-question presence (is exactly one nextQuestion emitted?)
|
|
*
|
|
* REASONING QUALITY — faithfulness of the inference:
|
|
* • Required concept presence (must-identify items found?)
|
|
* • Unsupported inference absence (prohibited claims genuinely absent?)
|
|
*
|
|
* A test case can pass technical but fail reasoning (hallucination),
|
|
* or pass reasoning but fail technical (missing fields, schema errors).
|
|
*/
|
|
|
|
import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs";
|
|
import { join, dirname } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
const __filename = fileURLToPath(import.meta.url);
|
|
const __dirname = dirname(__filename);
|
|
|
|
// ── Config ───────────────────────────────────────────
|
|
const useRealProvider = process.env.EVAL_REAL === "1";
|
|
const useDiagnostic = process.env.EVAL_DIAGNOSTIC === "1";
|
|
|
|
let testDataPath;
|
|
if (useDiagnostic) {
|
|
testDataPath = join(__dirname, "data", "live-diagnostic-v0.2.json");
|
|
} else {
|
|
testDataPath = join(__dirname, "test-data", "v0.2-evaluation.jsonl");
|
|
}
|
|
|
|
// Standard results dir (for full evals) vs live diagnostic results dir
|
|
const resultsDir = useDiagnostic
|
|
? join(__dirname, "..", "evaluation-results")
|
|
: join(__dirname, "..", "tests-results");
|
|
|
|
if (!existsSync(resultsDir)) {
|
|
mkdirSync(resultsDir, { recursive: true });
|
|
}
|
|
|
|
// ── Load test cases ──────────────────────────────────
|
|
function loadTestCases(path) {
|
|
const content = readFileSync(path, "utf-8");
|
|
// Support both JSONL (one JSON object per line) and JSON array formats
|
|
if (path.endsWith(".json")) {
|
|
return JSON.parse(content);
|
|
}
|
|
return content
|
|
.split("\n")
|
|
.filter((line) => line.trim())
|
|
.map((line) => JSON.parse(line));
|
|
}
|
|
|
|
// ── Normalise text for comparison ────────────────────
|
|
function normalise(text) {
|
|
return String(text)
|
|
.toLowerCase()
|
|
.replace(/[^\w\s_]/g, " ")
|
|
.replace(/\s+/g, " ")
|
|
.trim();
|
|
}
|
|
|
|
// ── Technical scoring helpers ────────────────────────
|
|
|
|
function checkPrimaryTypeMatch(actualPrimary, expectedTypes) {
|
|
if (!actualPrimary || !expectedTypes?.length) return false;
|
|
const actual = String(actualPrimary).toLowerCase().replace(/\s+/g, "_");
|
|
return expectedTypes.some((t) => t.toLowerCase().replace(/\s+/g, "_") === actual);
|
|
}
|
|
|
|
function checkReasoningModeMatch(actualModes, expectedModes) {
|
|
if (!actualModes?.length || !expectedModes?.length) return false;
|
|
const actual = actualModes.map((m) => String(m).toLowerCase().replace(/\s+/g, "_"));
|
|
const expected = expectedModes.map((m) => String(m).toLowerCase().replace(/\s+/g, "_"));
|
|
return expected.some((e) => actual.includes(e));
|
|
}
|
|
|
|
function checkNextQuestionPresent(nextQuestion) {
|
|
return nextQuestion !== null && nextQuestion !== undefined && nextQuestion !== "";
|
|
}
|
|
|
|
// ── Reasoning quality helpers ────────────────────────
|
|
|
|
function checkConceptPresence(actualText, concepts) {
|
|
if (!concepts?.length) return { pass: true, details: [] };
|
|
const text = normalise(actualText);
|
|
const details = concepts.map((c) => ({
|
|
concept: c,
|
|
found: text.includes(normalise(c)),
|
|
}));
|
|
return { pass: details.every((d) => d.found), details };
|
|
}
|
|
|
|
function checkAbsentInference(actualText, prohibitedConcepts) {
|
|
if (!prohibitedConcepts?.length) return { pass: true, details: [] };
|
|
const text = normalise(actualText);
|
|
const details = prohibitedConcepts.map((c) => ({
|
|
concept: c,
|
|
absent: !text.includes(normalise(c)),
|
|
}));
|
|
return { pass: details.every((d) => d.absent), details };
|
|
}
|
|
|
|
// ── Run a single test case ───────────────────────────
|
|
async function runTestCase(testCase, analyseScenarioFn) {
|
|
const base = {
|
|
id: testCase.id,
|
|
input: testCase.input.slice(0, 200),
|
|
responseDurationMs: 0,
|
|
actualPrimaryType: null,
|
|
actualReasoningModes: [],
|
|
};
|
|
|
|
// ── TECHNICAL result ────────────────────────────────
|
|
const technical = {
|
|
schemaValid: false,
|
|
classificationMatch: false,
|
|
reasoningModeMatch: false,
|
|
nextQuestionPresent: false,
|
|
pass: false,
|
|
errors: [],
|
|
};
|
|
|
|
// ── REASONING QUALITY result ────────────────────────
|
|
const reasoningQuality = {
|
|
requiredConcepts: { pass: true, details: [] },
|
|
unsupportedInferencesAbsent: { pass: true, details: [] },
|
|
pass: false,
|
|
};
|
|
|
|
try {
|
|
const analysisResult = await analyseScenarioFn(testCase.input, { promptVersion: "v0.2" });
|
|
|
|
base.responseDurationMs = analysisResult.responseDurationMs || 0;
|
|
base.rawOutput = analysisResult.rawResponse?.slice(0, 500);
|
|
|
|
if (analysisResult.success) {
|
|
technical.schemaValid = true;
|
|
const actualPrimary = analysisResult.inputClassification?.primaryType;
|
|
technical.classificationMatch = checkPrimaryTypeMatch(actualPrimary, testCase.expectedPrimaryTypes);
|
|
base.actualPrimaryType = actualPrimary;
|
|
|
|
const modes = analysisResult.inputClassification?.reasoningModes || [];
|
|
technical.reasoningModeMatch = checkReasoningModeMatch(modes, testCase.expectedReasoningModes);
|
|
base.actualReasoningModes = modes;
|
|
|
|
technical.nextQuestionPresent = checkNextQuestionPresent(analysisResult.nextQuestion);
|
|
|
|
// ── Reasoning quality checks ─────────────────────
|
|
const summaryText = analysisResult.reconstruction?.summary || "";
|
|
const evidenceTexts = (analysisResult.evidence || []).map((e) => e.description);
|
|
const allEvidenceRaw = (analysisResult.evidence || []).map(
|
|
(e) => `${e.description} ${e.attribution || ""}`
|
|
);
|
|
|
|
reasoningQuality.requiredConcepts = checkConceptPresence(
|
|
[summaryText, ...evidenceTexts].join(" "),
|
|
testCase.shouldIdentify
|
|
);
|
|
|
|
reasoningQuality.unsupportedInferencesAbsent = checkAbsentInference(
|
|
allEvidenceRaw.join(" "),
|
|
testCase.shouldNotInfer
|
|
);
|
|
|
|
// ── Combined pass criteria ───────────────────────
|
|
technical.pass =
|
|
technical.schemaValid && technical.classificationMatch && technical.nextQuestionPresent;
|
|
reasoningQuality.pass =
|
|
reasoningQuality.requiredConcepts.pass && reasoningQuality.unsupportedInferencesAbsent.pass;
|
|
} else {
|
|
technical.errors = analysisResult.errors || [analysisResult.error];
|
|
if (analysisResult.error) technical.errors.push(analysisResult.error);
|
|
}
|
|
} catch (e) {
|
|
technical.errors.push(e.message || String(e));
|
|
}
|
|
|
|
return { ...base, technical, reasoningQuality };
|
|
}
|
|
|
|
// ── Mock provider for evaluation ─────────────────────
|
|
class MockProvider {
|
|
constructor() {
|
|
this.name = "mock";
|
|
}
|
|
|
|
async generateReconstruction(prompt, modelName) {
|
|
// Extract the scenario text from the prompt template
|
|
let scenario = prompt;
|
|
const scenarioMarker = "Scenario:\n";
|
|
const markerIdx = prompt.indexOf(scenarioMarker);
|
|
if (markerIdx >= 0) {
|
|
scenario = prompt.slice(markerIdx + scenarioMarker.length).trim();
|
|
}
|
|
const instructionSeparator = "\n\nReturn ONLY";
|
|
const instIdx = scenario.indexOf(instructionSeparator);
|
|
if (instIdx >= 0) {
|
|
scenario = scenario.slice(0, instIdx).trim();
|
|
}
|
|
|
|
// ── Keyword detection on scenario text only ───────
|
|
const hasAllWord = /\ball\b|\bno one\b|\bevery\b/i.test(scenario);
|
|
const hasSomeWord = /\bsome\b/i.test(scenario);
|
|
const hasComplaints = /complaint/i.test(scenario);
|
|
const hasSales = /sales/i.test(scenario);
|
|
const hasRevenue = /revenue|profit|margin/i.test(scenario);
|
|
const hasReportedSpeaker = /\b(?:reported|said|claimed|stated)\b.*\b(?:cfo|warehouse manager|user|customer|team|analyst|regulator|operator)\b|\b(?:cfo|warehouse manager|user|customer|team|analyst|regulator|operator)\b.*\b(?:reported|said|claimed|stated)\b/i.test(scenario);
|
|
const hasContradictionSignal = /\bbut\b|\bwile\b|\bothers\s+say\b|\bis better.*is slower\b/i.test(scenario);
|
|
const hasChangeIndicator = /\b(?:increased|decreased|fell|dropped|grew|rose|declined|up by |down by |changed from |went from |tripled|doubled|halved)\b/i.test(scenario);
|
|
const hasDecisionRequest = /\b(?:need\s+to\s+improve|need\s+better|we should implement|should fix|want .* launch.*market|launch .* app.*capture|implement .* because.*competitor)\b/i.test(scenario);
|
|
const hasAmbiguous = /philosophical|therefore i am|ambiguous statement|meta.?context/i.test(scenario);
|
|
const hasCausalSignal = /\bafter\b.*(?:complaint|failure|issue|problem|price|deployment)|deployed.*and.*(tripl|double|increase)|due to|\bbecause\b/i.test(scenario);
|
|
const hasTemporalComparison = /last month.*this month|was \d+.*\bby \d+%|\bfrom \d+.*to \d+|\b\d+% from \d+/.test(scenario);
|
|
const hasUnexpectedContinuity = /\bchanged.*but.*still|\bstill.*working/i.test(scenario);
|
|
|
|
// ── Classification hierarchy (most specific first) ─
|
|
let primaryType = "other";
|
|
|
|
if (hasAmbiguous) {
|
|
primaryType = "ambiguous_statement";
|
|
} else if (/^\s*I used the phrase/i.test(scenario)) {
|
|
primaryType = "question";
|
|
} else if (hasDecisionRequest || /\bneeds?\s+better|\bwe need to\b/i.test(scenario)) {
|
|
primaryType = "decision_request";
|
|
} else if (hasCausalSignal && hasSales) {
|
|
primaryType = "causal_claim";
|
|
} else if (hasCausalSignal && !hasRevenue) {
|
|
primaryType = "causal_claim";
|
|
} else if (hasContradictionSignal && hasRevenue) {
|
|
primaryType = "contradiction";
|
|
} else if (hasContradictionSignal && hasChangeIndicator) {
|
|
primaryType = "contradiction";
|
|
} else if (hasReportedSpeaker && !hasChangeIndicator) {
|
|
primaryType = "reported_claim";
|
|
} else if (hasUnexpectedContinuity) {
|
|
primaryType = "unexplained_change";
|
|
} else if (hasTemporalComparison && !hasRevenue) {
|
|
primaryType = "unexplained_change";
|
|
} else if (hasChangeIndicator && !hasAllWord && !hasSomeWord) {
|
|
primaryType = "unexplained_change";
|
|
} else if (hasChangeIndicator && hasRevenue) {
|
|
primaryType = "unexplained_change";
|
|
} else if (hasAllWord || hasSales) {
|
|
primaryType = "observed_problem";
|
|
} else if (hasSomeWord && !hasAllWord) {
|
|
primaryType = "observed_problem";
|
|
} else if (hasChangeIndicator || hasComplaints) {
|
|
primaryType = "unexplained_change";
|
|
} else if (/^[A-Z]/.test(scenario.trim())) {
|
|
primaryType = "observed_problem";
|
|
}
|
|
|
|
const secondaryTypes = [];
|
|
if (primaryType === "observed_problem") secondaryTypes.push("fault_report");
|
|
if (hasComplaints || hasSales) secondaryTypes.push("unexplained_change");
|
|
|
|
const reasoningModes = ["identify_difference"];
|
|
if (primaryType === "contradiction") reasoningModes.unshift("investigate_contradiction");
|
|
if (primaryType === "decision_request" || primaryType === "desired_outcome") {
|
|
reasoningModes.push("decision_support", "identify_missing_information");
|
|
}
|
|
if (hasComplaints || hasSales) {
|
|
if (!reasoningModes.includes("establish_baseline")) {
|
|
reasoningModes.unshift("establish_baseline");
|
|
}
|
|
}
|
|
if (hasAmbiguous) reasoningModes.push("clarify_meaning");
|
|
if (primaryType === "reported_claim") reasoningModes.push("validate_claim");
|
|
if (!secondaryTypes.includes("unexplained_change") && primaryType === "unexplained_change") {
|
|
reasoningModes.push("establish_baseline", "validate_measurement");
|
|
}
|
|
|
|
return {
|
|
inputClassification: {
|
|
primaryType,
|
|
secondaryTypes,
|
|
reasoningModes,
|
|
classificationReason: `Analyzing ${primaryType} with secondary types: ${secondaryTypes.join(", ") || "none"}. Input was evaluated for operational anchors including actors, states, differences, and evidence sources.`,
|
|
confidence: hasComplaints ? "high" : "medium",
|
|
},
|
|
reconstruction: {
|
|
summary: `${primaryType.charAt(0).toUpperCase() + primaryType.slice(1)} detected in input. The scenario involves ${hasComplaints ? "reported complaints" : hasSales ? "declining metrics" : "observed operational context"} that warrants further investigation to establish baseline and identify key differences.`,
|
|
actors: [],
|
|
systemsOrObjects: [],
|
|
expectedStates: [],
|
|
observedStates: [],
|
|
differences: [hasSomeWord ? { id: "d1", description: "The input contains a subset modifier ('some'), indicating not universal applicability", confidence: "high", importance: "important" } : { id: "d1", description: "Key operational distinction identified in the scenario data", confidence: "medium", importance: "supporting" }],
|
|
knownTransitions: [],
|
|
unexplainedTransitions: [],
|
|
contradictions: hasContradictionSignal ? [{ id: "c1", description: "Divergent signals detected between reported metrics and contextual anchors", confidence: "medium", importance: "important" }] : [],
|
|
importantUnknowns: [hasComplaints ? { id: "u1", description: "Baseline period and absolute numbers for the complaint change", confidence: "high", importance: "critical" } : { id: "u1", description: "Contextual anchors needed to establish operational significance", confidence: "medium", importance: "supporting" }],
|
|
plausibleInterpretations: [{ id: "pi1", description: "The situation represents a genuine operational issue requiring investigation", supportingEvidenceIds: ["d1"], assumptionsRequired: ["input contains meaningful operational content"], confidence: "medium" }],
|
|
},
|
|
evidence: [
|
|
{ id: "e1", description: "Primary operational indicator detected in input text", evidenceType: "direct_observation", confidence: "high", importance: "supporting" },
|
|
],
|
|
nextQuestion: {
|
|
id: "q1",
|
|
question: hasComplaints ? "What is the baseline number of complaints and over what time period?" : "What specific metric or state should be used as the reference point?",
|
|
targets: ["baseline_context", "measurement_period"],
|
|
reason: "Establishing a reference point would distinguish whether the reported change is significant or within normal variation.",
|
|
expectedInformationValue: "high",
|
|
reasoningMode: "establish_baseline",
|
|
},
|
|
};
|
|
}
|
|
}
|
|
|
|
// ── Display helpers ──────────────────────────────────
|
|
|
|
const CATEGORY_COLORS = {
|
|
technical: "\x1b[36m", // cyan
|
|
reasoning: "\x1b[33m", // yellow
|
|
reset: "\x1b[0m",
|
|
};
|
|
|
|
function categoryLabel(label) {
|
|
return `${CATEGORY_COLORS.technical}${label}${CATEGORY_COLORS.reset}`;
|
|
}
|
|
|
|
function reasonCategoryLabel() {
|
|
return `${CATEGORY_COLORS.reasoning}reasoning quality${CATEGORY_COLORS.reset}`;
|
|
}
|
|
|
|
// ── Main evaluation loop ─────────────────────────────
|
|
async function main() {
|
|
const testCases = loadTestCases(testDataPath);
|
|
console.log(`\n⚡ Confidence Engine v0.2 — Evaluation Harness`);
|
|
console.log(` Provider: ${useRealProvider ? "Ollama (real)" : "Mock"}`);
|
|
console.log(` Cases loaded: ${testCases.length}\n`);
|
|
|
|
// Import or instantiate analysis function
|
|
let analyseScenarioFn;
|
|
if (useRealProvider) {
|
|
const { analyseScenario } = await import("../lib/analysis.js");
|
|
analyseScenarioFn = analyseScenario;
|
|
} else {
|
|
const mockProvider = new MockProvider();
|
|
const schemaMod = await import("../lib/reconstruction/schema.js");
|
|
const { reconstructionV2Schema, reconstructionSchema: reconstructionV1Schema } = schemaMod;
|
|
const { buildPrompt } = await import("../lib/reconstruction/prompt.js");
|
|
|
|
analyseScenarioFn = async (scenario, opts = {}) => {
|
|
const startTime = Date.now();
|
|
const trimmed = scenario.trim();
|
|
if (!trimmed) return { success: false, error: "Empty scenario", responseDurationMs: 0 };
|
|
|
|
let promptObj;
|
|
try {
|
|
promptObj = await buildPrompt(trimmed, opts.promptVersion || "v0.2");
|
|
} catch {
|
|
promptObj = { prompt: trimmed, version: "v0.2" };
|
|
}
|
|
|
|
const mockResult = await mockProvider.generateReconstruction(promptObj.prompt, process.env.OLLAMA_MODEL || "mock-model");
|
|
|
|
let schemaValid = false;
|
|
let validatedData = null;
|
|
if (reconstructionV2Schema.safeParse) {
|
|
const v2Result = reconstructionV2Schema.safeParse(mockResult);
|
|
if (v2Result.success) {
|
|
schemaValid = true;
|
|
validatedData = v2Result.data;
|
|
} else {
|
|
const v1Result = reconstructionV1Schema.safeParse(mockResult);
|
|
if (v1Result.success) {
|
|
schemaValid = true;
|
|
validatedData = v1Result.data;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!schemaValid || !validatedData) {
|
|
return {
|
|
success: false,
|
|
validationStatus: "invalid",
|
|
modelName: "mock-model",
|
|
responseDurationMs: Date.now() - startTime,
|
|
promptVersion: opts.promptVersion || "v0.2",
|
|
reconstruction: null,
|
|
};
|
|
}
|
|
|
|
return {
|
|
success: true,
|
|
validationStatus: "valid",
|
|
modelName: "mock-model",
|
|
responseDurationMs: Date.now() - startTime,
|
|
promptVersion: opts.promptVersion || "v0.2",
|
|
inputClassification: validatedData.inputClassification,
|
|
reconstruction: validatedData.reconstruction,
|
|
evidence: validatedData.evidence,
|
|
nextQuestion: validatedData.nextQuestion,
|
|
};
|
|
};
|
|
}
|
|
|
|
// Run all cases
|
|
const results = [];
|
|
for (const tc of testCases) {
|
|
process.stdout.write(` ${tc.id}: ... `);
|
|
const r = await runTestCase(tc, analyseScenarioFn);
|
|
results.push(r);
|
|
const tStatus = r.technical.pass ? "\x1b[32m✅\x1b[0m" : "\x1b[31m❌\x1b[0m"; // green / red
|
|
const rqStatus = r.reasoningQuality.pass ? "\x1b[32m✅\x1b[0m" : "\x1b[31m❌\x1b[0m";
|
|
|
|
process.stdout.write(`${tStatus} tech ${rqStatus} reason\n`);
|
|
if (!r.technical.pass && r.technical.errors?.length) {
|
|
for (const e of r.technical.errors.slice(0, 2)) process.stdout.write(` → [tech] ${e}\n`);
|
|
} else if (!r.technical.pass) {
|
|
const reasons = [];
|
|
if (!r.technical.schemaValid) reasons.push("schema invalid");
|
|
if (!r.technical.classificationMatch) reasons.push("classification mismatch");
|
|
if (!r.technical.nextQuestionPresent) reasons.push("no next question");
|
|
process.stdout.write(` → [tech] ${reasons.join(", ")}\n`);
|
|
}
|
|
|
|
if (!r.reasoningQuality.pass) {
|
|
const rqReasons = [];
|
|
if (!r.reasoningQuality.requiredConcepts.pass) {
|
|
rqReasons.push("missing required concept(s)");
|
|
}
|
|
if (!r.reasoningQuality.unsupportedInferencesAbsent.pass) {
|
|
rqReasons.push("unsupported inference present");
|
|
}
|
|
process.stdout.write(` → [reasoning] ${rqReasons.join(", ")}\n`);
|
|
}
|
|
}
|
|
|
|
// ── Compute summary stats ────────────────────────────
|
|
const total = results.length;
|
|
const techPassCount = results.filter((r) => r.technical.pass).length;
|
|
const techSchemaValidCount = results.filter((r) => r.technical.schemaValid).length;
|
|
const techClassificationMatchCount = results.filter((r) => r.technical.classificationMatch).length;
|
|
const techNextQuestionPresentCount = results.filter((r) => r.technical.nextQuestionPresent).length;
|
|
|
|
const rqPassCount = results.filter((r) => r.reasoningQuality.pass).length;
|
|
const rqConceptsPassCount = results.filter((r) => r.reasoningQuality.requiredConcepts.pass).length;
|
|
const rqAbsencePassCount = results.filter((r) => r.reasoningQuality.unsupportedInferencesAbsent.pass).length;
|
|
|
|
const anyPassCount = results.filter(
|
|
(r) => r.technical.pass && r.reasoningQuality.pass
|
|
).length;
|
|
|
|
const avgDuration = total > 0
|
|
? results.reduce((s, r) => s + (r.responseDurationMs || 0), 0) / total
|
|
: 0;
|
|
|
|
const failedTechCases = results.filter((r) => !r.technical.pass);
|
|
const failedRqCases = results.filter((r) => !r.reasoningQuality.pass);
|
|
const techPassOnly = results.filter(
|
|
(r) => r.technical.pass && !r.reasoningQuality.pass
|
|
);
|
|
const rqPassOnly = results.filter(
|
|
(r) => !r.technical.pass && r.reasoningQuality.pass
|
|
);
|
|
|
|
// ── Console summary ───────────────────────────────────
|
|
console.log(`\n${"=".repeat(60)}`);
|
|
console.log("EVALUATION SUMMARY");
|
|
console.log(`${"=".repeat(60)}\n`);
|
|
|
|
console.log(`Cases run: ${total}\n`);
|
|
|
|
// Technical section
|
|
console.log(categoryLabel("─── TECHNICAL ──────────────────────────────"));
|
|
console.log(` Schema validity rate: ${techSchemaValidCount}/${total} ${(techSchemaValidCount / total * 100).toFixed(1)}%`);
|
|
console.log(` Classification match: ${techClassificationMatchCount}/${total} ${(techClassificationMatchCount / total * 100).toFixed(1)}%`);
|
|
console.log(` Next-question present: ${techNextQuestionPresentCount}/${total} ${(techNextQuestionPresentCount / total * 100).toFixed(1)}%`);
|
|
console.log(` Technical pass rate: ${techPassCount}/${total} ${(techPassCount / total * 100).toFixed(1)}%\n`);
|
|
|
|
// Reasoning quality section
|
|
console.log(reasonCategoryLabel() + " ─────────────────────────────");
|
|
console.log(`${CATEGORY_COLORS.reset}`);
|
|
console.log(` Required concept match: ${rqConceptsPassCount}/${total} ${(rqConceptsPassCount / total * 100).toFixed(1)}%`);
|
|
console.log(` Unsupported inference absent: ${rqAbsencePassCount}/${total} ${(rqAbsencePassCount / total * 100).toFixed(1)}%`);
|
|
console.log(` Reasoning quality pass: ${rqPassCount}/${total} ${(rqPassCount / total * 100).toFixed(1)}%\n`);
|
|
|
|
// Combined
|
|
console.log(`${"─".repeat(60)}`);
|
|
console.log(` Both technical + reasoning: ${anyPassCount}/${total} ${(anyPassCount / total * 100).toFixed(1)}%`);
|
|
if (techPassOnly.length > 0) {
|
|
console.log(` Technical only (hallucinated): ${techPassOnly.length} — IDs: ${techPassOnly.map((r) => r.id).join(", ")}`);
|
|
}
|
|
if (rqPassOnly.length > 0) {
|
|
console.log(` Reasoning only (bad structure): ${rqPassOnly.length} — IDs: ${rqPassOnly.map((r) => r.id).join(", ")}`);
|
|
}
|
|
if (failedTechCases.length > 0 && failedRqCases.length > 0) {
|
|
console.log(` Failed both: ${results.filter((r) => !r.technical.pass && !r.reasoningQuality.pass).length}`);
|
|
}
|
|
|
|
console.log(` Avg response duration: ${avgDuration.toFixed(0)}ms`);
|
|
console.log(`${"=".repeat(60)}\n`);
|
|
|
|
if (failedTechCases.length > 0) {
|
|
console.log(`Failed technical — case IDs: ${failedTechCases.map((r) => r.id).join(", ")}`);
|
|
}
|
|
if (failedRqCases.length > 0) {
|
|
console.log(`Failed reasoning quality — case IDs: ${failedRqCases.map((r) => r.id).join(", ")}`);
|
|
}
|
|
|
|
// ── Save results ──────────────────────────────────────
|
|
const timestamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
|
|
|
if (useDiagnostic) {
|
|
// Live diagnostic: save to a dedicated result directory with per-case files + summary
|
|
const caseResultDir = join(resultsDir, timestamp);
|
|
mkdirSync(caseResultDir, { recursive: true });
|
|
|
|
// Per-case results JSON + Markdown
|
|
for (const r of results) {
|
|
const tc = testCases.find((t) => t.id === r.id);
|
|
const caseFileBase = join(caseResultDir, r.id);
|
|
|
|
// Raw case result JSON
|
|
writeFileSync(
|
|
`${caseFileBase}-result.json`,
|
|
JSON.stringify({
|
|
id: r.id,
|
|
description: tc?.description || "",
|
|
input: tc?.input,
|
|
responseDurationMs: r.responseDurationMs,
|
|
actualPrimaryType: r.actualPrimaryType,
|
|
actualReasoningModes: r.actualReasoningModes,
|
|
rawOutput: r.rawOutput,
|
|
technical: r.technical,
|
|
reasoningQuality: r.reasoningQuality,
|
|
}, null, 2)
|
|
);
|
|
|
|
// Per-case Markdown summary
|
|
const techStatus = r.technical.pass ? "✅ PASS" : "❌ FAIL";
|
|
const rqStatus = r.reasoningQuality.pass ? "✅ PASS" : "❌ FAIL";
|
|
|
|
let md = `# Diagnostic Case: ${r.id}\n\n`;
|
|
md += `${tc?.description || ""}\n\n`;
|
|
md += `## Input\n\n\`\`\`\n${tc?.input || r.input}\n\`\`\`\n\n`;
|
|
md += `## Result\n\n`;
|
|
md += `- **Technical**: ${techStatus} (${(r.technical.pass ? 1 : 0)}/${Object.keys(r.technical).filter(k => typeof r.technical[k] === "boolean" && k !== "pass").length} sub-checks pass)\n`;
|
|
md += `- **Reasoning Quality**: ${rqStatus} (${(r.reasoningQuality.pass ? 1 : 0)}/${2} sub-checks pass)\n`;
|
|
md += `- **Actual Primary Type**: ${r.actualPrimaryType || "N/A"}\n`;
|
|
md += `- **Actual Reasoning Modes**: ${(r.actualReasoningModes || []).join(", ") || "N/A"}\n`;
|
|
md += `- **Response Duration**: ${r.responseDurationMs}ms\n`;
|
|
|
|
if (!r.technical.pass) {
|
|
const reasons = [];
|
|
if (!r.technical.schemaValid) reasons.push("schema invalid");
|
|
if (!r.technical.classificationMatch) reasons.push("classification mismatch");
|
|
if (!r.technical.nextQuestionPresent) reasons.push("no next question");
|
|
md += `\n### Technical Failures\n\n${reasons.join(", ")}\n`;
|
|
}
|
|
|
|
if (!r.reasoningQuality.pass) {
|
|
const rqReasons = [];
|
|
if (!r.reasoningQuality.requiredConcepts.pass) {
|
|
rqReasons.push("missing required concept(s): " + r.reasoningQuality.requiredConcepts.details.filter(d => !d.found).map(d => d.concept).join(", ") || "unknown");
|
|
}
|
|
if (!r.reasoningQuality.unsupportedInferencesAbsent.pass) {
|
|
rqReasons.push("unsupported inference present: " + r.reasoningQuality.unsupportedInferencesAbsent.details.filter(d => !d.absent).map(d => d.concept).join(", ") || "unknown");
|
|
}
|
|
md += `\n### Reasoning Quality Failures\n\n${rqReasons.join("\n")}\n`;
|
|
}
|
|
|
|
writeFileSync(`${caseFileBase}-summary.md`, md);
|
|
}
|
|
|
|
// Directory-level summary JSON
|
|
const fullResults = {
|
|
timestamp: new Date().toISOString(),
|
|
provider: useRealProvider ? "ollama-real" : "mock",
|
|
promptVersion: "v0.2",
|
|
casesRun: total,
|
|
summary: {
|
|
technical: {
|
|
schemaValidityRate: `${(techSchemaValidCount / total * 100).toFixed(1)}%`,
|
|
classificationMatchRate: `${(techClassificationMatchCount / total * 100).toFixed(1)}%`,
|
|
nextQuestionPresentRate: `${(techNextQuestionPresentCount / total * 100).toFixed(1)}%`,
|
|
passRate: `${(techPassCount / total * 100).toFixed(1)}%`,
|
|
},
|
|
reasoningQuality: {
|
|
requiredConceptMatchRate: `${(rqConceptsPassCount / total * 100).toFixed(1)}%`,
|
|
unsupportedInferenceFailures: (total - rqAbsencePassCount).toString(),
|
|
passRate: `${(rqPassCount / total * 100).toFixed(1)}%`,
|
|
},
|
|
combinedPassRate: `${(anyPassCount / total * 100).toFixed(1)}%`,
|
|
averageResponseDurationMs: avgDuration.toFixed(0),
|
|
},
|
|
testCaseResults: results.map((r) => ({
|
|
id: r.id,
|
|
input: r.input,
|
|
responseDurationMs: r.responseDurationMs,
|
|
actualPrimaryType: r.actualPrimaryType,
|
|
actualReasoningModes: r.actualReasoningModes,
|
|
technical: {
|
|
schemaValid: r.technical.schemaValid,
|
|
classificationMatch: r.technical.classificationMatch,
|
|
reasoningModeMatch: r.technical.reasoningModeMatch,
|
|
nextQuestionPresent: r.technical.nextQuestionPresent,
|
|
pass: r.technical.pass,
|
|
errors: r.technical.errors,
|
|
},
|
|
reasoningQuality: {
|
|
requiredConcepts: r.reasoningQuality.requiredConcepts,
|
|
unsupportedInferencesAbsent: r.reasoningQuality.unsupportedInferencesAbsent,
|
|
pass: r.reasoningQuality.pass,
|
|
},
|
|
})),
|
|
};
|
|
|
|
writeFileSync(join(caseResultDir, "summary.json"), JSON.stringify(fullResults, null, 2));
|
|
console.log(`Live diagnostic results saved to: ${caseResultDir}/`);
|
|
|
|
// Also save a top-level manifest pointing to the latest run
|
|
const manifestPath = join(resultsDir, "latest-manifest.json");
|
|
writeFileSync(manifestPath, JSON.stringify({ latestRun: timestamp, caseCount: total }, null, 2));
|
|
console.log(`Manifest saved to: ${manifestPath}`);
|
|
|
|
} else {
|
|
// Standard (non-diagnostic): single file output
|
|
const resultsFile = join(resultsDir, `evaluation-${timestamp}.json`);
|
|
const fullResults = {
|
|
timestamp: new Date().toISOString(),
|
|
provider: useRealProvider ? "ollama-real" : "mock",
|
|
promptVersion: "v0.2",
|
|
casesRun: total,
|
|
summary: {
|
|
technical: {
|
|
schemaValidityRate: `${(techSchemaValidCount / total * 100).toFixed(1)}%`,
|
|
classificationMatchRate: `${(techClassificationMatchCount / total * 100).toFixed(1)}%`,
|
|
nextQuestionPresentRate: `${(techNextQuestionPresentCount / total * 100).toFixed(1)}%`,
|
|
passRate: `${(techPassCount / total * 100).toFixed(1)}%`,
|
|
},
|
|
reasoningQuality: {
|
|
requiredConceptMatchRate: `${(rqConceptsPassCount / total * 100).toFixed(1)}%`,
|
|
unsupportedInferenceFailures: (total - rqAbsencePassCount).toString(),
|
|
passRate: `${(rqPassCount / total * 100).toFixed(1)}%`,
|
|
},
|
|
combinedPassRate: `${(anyPassCount / total * 100).toFixed(1)}%`,
|
|
averageResponseDurationMs: avgDuration.toFixed(0),
|
|
},
|
|
testCaseResults: results.map((r) => ({
|
|
id: r.id,
|
|
input: r.input,
|
|
responseDurationMs: r.responseDurationMs,
|
|
actualPrimaryType: r.actualPrimaryType,
|
|
actualReasoningModes: r.actualReasoningModes,
|
|
technical: {
|
|
schemaValid: r.technical.schemaValid,
|
|
classificationMatch: r.technical.classificationMatch,
|
|
reasoningModeMatch: r.technical.reasoningModeMatch,
|
|
nextQuestionPresent: r.technical.nextQuestionPresent,
|
|
pass: r.technical.pass,
|
|
errors: r.technical.errors,
|
|
},
|
|
reasoningQuality: {
|
|
requiredConcepts: r.reasoningQuality.requiredConcepts,
|
|
unsupportedInferencesAbsent: r.reasoningQuality.unsupportedInferencesAbsent,
|
|
pass: r.reasoningQuality.pass,
|
|
},
|
|
})),
|
|
};
|
|
|
|
writeFileSync(resultsFile, JSON.stringify(fullResults, null, 2));
|
|
console.log(`Results saved to: ${resultsFile}`);
|
|
console.log(`${"=".repeat(60)}\n`);
|
|
}
|
|
|
|
// ── Close main() scope if we're in the non-diagnostic branch ──
|
|
// (The if/else above handles result saving; main closes here)
|
|
}
|
|
|
|
main().catch((e) => {
|
|
console.error("Evaluator failed:", e.message);
|
|
process.exit(1);
|
|
});
|