robbond
a0bcb12792
Core fix: For cases with expectedBehaviours, reasoningQuality.status is now set exclusively from behaviour evaluation results (required behaviour pass/fail). Legacy concept checks remain visible as diagnostic-only metrics and do not influence the authoritative result. Key changes: - Behaviour-based scoring determines reasoning status (passed/failed) instead of legacy concept literal matching - Schema failure correctly forces not_evaluated (no vacuous truth) - Saved live results re-evaluator preserves provenance metadata - Classification tolerance map works bidirectionally for interchangeable types - normalise() treats underscores as word characters, hyphens as spaces Tests: 74 passing across both evaluator test suites - tests/evaluator-behaviour-authoritative.test.mjs (47 tests, new) - tests/evaluator-semantic.test.mjs (27 tests)
2126 lines
74 KiB
JavaScript
Executable File
2126 lines
74 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
|
/**
|
|
* Evaluation harness for Confidence Engine v0.2 — semantic reasoning evaluator.
|
|
*
|
|
* Measures reasoning behaviour rather than exact wording. Uses:
|
|
* • Multiple accepted classifications per case (classification tolerance)
|
|
* • Behaviour expectations with accepted signals (not literal phrases)
|
|
* • Proper separation of schema failure from reasoning evaluation
|
|
* • Evidence-type alias normalisation with diagnostics logging
|
|
* • Structured output field inspection alongside text matching
|
|
*
|
|
* Scoring categories:
|
|
* TECHNICAL — structural correctness of the output:
|
|
* • Schema validity (does the JSON match the schema?)
|
|
* • Classification accuracy (primary type among accepted types? reasoning modes present?)
|
|
* • Next-question presence (is a nextQuestion emitted?)
|
|
*
|
|
* REASONING QUALITY — faithfulness of the inference:
|
|
* • Required concept presence (legacy field, kept for backward compat)
|
|
* • Unsupported inference absence (prohibited claims genuinely absent?)
|
|
* • Expected behaviour coverage (semantic matching across multiple signal types)
|
|
*
|
|
* A test case can pass technical but fail reasoning (hallucination),
|
|
* or pass reasoning but fail technical (missing fields, schema errors).
|
|
* If schema fails: reasoning is marked 'not_evaluated' — no vacuous truth.
|
|
*/
|
|
|
|
import {
|
|
readFileSync,
|
|
writeFileSync,
|
|
mkdirSync,
|
|
existsSync,
|
|
readdirSync,
|
|
statSync,
|
|
} from "node:fs";
|
|
import { join, dirname } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
const __filename = fileURLToPath(import.meta.url);
|
|
const __dirname = dirname(__filename);
|
|
|
|
// ── Config modes ─────────────────────────────────────
|
|
const useRealProvider = process.env.EVAL_REAL === "1";
|
|
const useDiagnostic = process.env.EVAL_DIAGNOSTIC === "1";
|
|
const savedResultsDir = process.env.EVAL_SAVED_RESULTS;
|
|
let mode = "normal"; // normal | diagnostic | saved
|
|
|
|
if (savedResultsDir) {
|
|
mode = "saved";
|
|
} else if (useDiagnostic) {
|
|
mode = "diagnostic";
|
|
}
|
|
|
|
let testDataPath;
|
|
if (mode === "diagnostic") {
|
|
testDataPath = join(__dirname, "data", "live-diagnostic-v0.2.json");
|
|
} else if (mode === "saved") {
|
|
testDataPath = null;
|
|
} else {
|
|
testDataPath = join(__dirname, "test-data", "v0.2-evaluation.jsonl");
|
|
}
|
|
|
|
const resultsDir =
|
|
mode === "diagnostic"
|
|
? join(__dirname, "..", "evaluation-results")
|
|
: mode === "saved"
|
|
? savedResultsDir
|
|
: join(__dirname, "..", "tests-results");
|
|
|
|
if (!existsSync(resultsDir)) mkdirSync(resultsDir, { recursive: true });
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// SYNONYM / SIGNAL GROUPS FOR SEMANTIC MATCHING
|
|
// ═══════════════════════════════════════════════════════
|
|
|
|
const SYN_G = {
|
|
baseline: [
|
|
"baseline",
|
|
"previous period",
|
|
"last month",
|
|
"before",
|
|
"normal level",
|
|
"comparison period",
|
|
"prior state",
|
|
"previous state",
|
|
"original",
|
|
"historical",
|
|
"pre-",
|
|
"formerly",
|
|
"initially",
|
|
],
|
|
subset: [
|
|
"some",
|
|
"subset",
|
|
"partial",
|
|
"certain users",
|
|
"not universal",
|
|
"limited to",
|
|
"only a few",
|
|
"a number of",
|
|
"several",
|
|
],
|
|
metricNorm: [
|
|
"normalise",
|
|
"denominator",
|
|
"rate",
|
|
"comparable scale",
|
|
"per unit",
|
|
"absolute vs relative",
|
|
"per customer",
|
|
"per transaction",
|
|
"basis points",
|
|
],
|
|
contra: [
|
|
"contradiction",
|
|
"divergent",
|
|
"opposing",
|
|
"conflicting",
|
|
"contrary to",
|
|
"but",
|
|
"however",
|
|
"yet",
|
|
"in contrast",
|
|
"despite",
|
|
"conversely",
|
|
],
|
|
trans: [
|
|
"transition",
|
|
"change from",
|
|
"before to",
|
|
"moved from",
|
|
"shifted",
|
|
"after",
|
|
"since",
|
|
"following",
|
|
"subsequent to",
|
|
"temporal sequence",
|
|
],
|
|
claimVal: [
|
|
"validate",
|
|
"corroborate",
|
|
"verify",
|
|
"confirm",
|
|
"evidence needed",
|
|
"single report",
|
|
"one user",
|
|
"anecdotal",
|
|
"unverified",
|
|
"claim",
|
|
],
|
|
};
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// NORMALISE + MATCHERS (deterministic, inspectable)
|
|
// ═══════════════════════════════════════════════════════
|
|
|
|
function normalise(text) {
|
|
return String(text)
|
|
.toLowerCase()
|
|
.replace(/[^\w\s_]/g, " ")
|
|
.replace(/\s+/g, " ")
|
|
.trim();
|
|
}
|
|
|
|
function matchesAnyPhrase(text, signals) {
|
|
if (!signals?.length || !text) return false;
|
|
const norm = normalise(text);
|
|
return signals.some((s) => norm.includes(normalise(s)));
|
|
}
|
|
|
|
function matchesReasoningMode(actualModes, acceptedModes) {
|
|
if (!acceptedModes?.length) return false;
|
|
const actual = (actualModes || []).map((m) =>
|
|
String(m).toLowerCase().replace(/\s+/g, "_"),
|
|
);
|
|
const normA = acceptedModes.map((m) =>
|
|
String(m).toLowerCase().replace(/\s+/g, "_"),
|
|
);
|
|
return normA.some((a) => actual.includes(a));
|
|
}
|
|
|
|
function matchesClassification(actualPrimary, acceptedTypes) {
|
|
if (!acceptedTypes?.length || !actualPrimary) return false;
|
|
const a = String(actualPrimary).toLowerCase().replace(/\s+/g, "_");
|
|
const n = acceptedTypes.map((t) =>
|
|
String(t).toLowerCase().replace(/\s+/g, "_"),
|
|
);
|
|
return n.includes(a);
|
|
}
|
|
|
|
function matchesSecondaryClassification(actualSecondary, acceptedTypes) {
|
|
if (!acceptedTypes?.length || !actualSecondary?.length) return false;
|
|
const actual = actualSecondary.map((t) =>
|
|
String(t).toLowerCase().replace(/\s+/g, "_"),
|
|
);
|
|
const normA = acceptedTypes.map((t) =>
|
|
String(t).toLowerCase().replace(/\s+/g, "_"),
|
|
);
|
|
return normA.some((a) => actual.includes(a));
|
|
}
|
|
|
|
function matchesStructuredField(output, fieldPath, signals) {
|
|
if (!output || !fieldPath?.length || !signals?.length) return false;
|
|
const parts = fieldPath.split(".");
|
|
let value = output;
|
|
for (const p of parts) {
|
|
if (value == null) return false;
|
|
value = value[p];
|
|
}
|
|
if (!Array.isArray(value)) {
|
|
if (typeof value === "object") {
|
|
const descs = [];
|
|
for (const k of ["description", "summary", "reason"]) {
|
|
if (typeof value[k] === "string") descs.push(value[k]);
|
|
}
|
|
return matchesAnyPhrase(descs.join(" "), signals);
|
|
}
|
|
return false;
|
|
}
|
|
const all = [];
|
|
for (const item of value) {
|
|
if (typeof item === "object" && item !== null) {
|
|
for (const k of ["description", "summary", "reason"]) {
|
|
if (typeof item[k] === "string") all.push(item[k]);
|
|
}
|
|
} else if (typeof item === "string") {
|
|
all.push(item);
|
|
}
|
|
}
|
|
return matchesAnyPhrase(all.join(" "), signals);
|
|
}
|
|
|
|
function checksImportantUnknowns(output, acceptedSignals) {
|
|
const unknowns = output?.importantUnknowns || [];
|
|
if (!unknowns.length) return false;
|
|
return matchesAnyPhrase(
|
|
unknowns.map((u) => u.description || "").join(" "),
|
|
acceptedSignals,
|
|
);
|
|
}
|
|
|
|
function checksNextQuestionTarget(output, acceptedSignals) {
|
|
if (!output?.nextQuestion) return false;
|
|
const q = output.nextQuestion;
|
|
const textParts = [q.question, q.reason].filter(Boolean).join(" ");
|
|
const targetText = (q.targets || []).map(String).join(" ");
|
|
return matchesAnyPhrase(`${textParts} ${targetText}`, acceptedSignals);
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// EVIDENCE NORMALISATION + VALIDATION
|
|
// ═══════════════════════════════════════════════════════
|
|
|
|
const VALID_EVIDENCE_TYPES = [
|
|
"direct_observation",
|
|
"reported_statement",
|
|
"interpretation",
|
|
"assumption",
|
|
"inferred_relationship",
|
|
];
|
|
const EVIDENCE_ALIASES = { reported_claim: "reported_statement" };
|
|
|
|
function normaliseEvidence(evidenceArray, diagnostics) {
|
|
const result = [];
|
|
let nullRemoved = 0;
|
|
let aliasChanged = 0;
|
|
let invalidTypes = [];
|
|
|
|
if (!Array.isArray(evidenceArray))
|
|
return { data: [], nullRemoved: 0, aliasChanged: 0, invalidTypes: [] };
|
|
|
|
for (const entry of evidenceArray) {
|
|
// Remove null entries with logging
|
|
if (entry === null || entry === undefined) {
|
|
nullRemoved++;
|
|
continue;
|
|
}
|
|
|
|
// Normalise evidence type aliases
|
|
let e = { ...entry };
|
|
if (e.evidenceType && EVIDENCE_ALIASES[e.evidenceType]) {
|
|
const original = e.evidenceType;
|
|
e.evidenceType = EVIDENCE_ALIASES[e.evidenceType];
|
|
aliasChanged++;
|
|
invalidTypes.push({ removed: null, normalised: null }); // placeholder
|
|
}
|
|
|
|
// Reject invalid evidence types with clear diagnostic info
|
|
if (e.evidenceType && !VALID_EVIDENCE_TYPES.includes(e.evidenceType)) {
|
|
diagnostics.push({
|
|
action: "invalid_evidence_type",
|
|
originalEvidenceType: e.evidenceType,
|
|
validTypes: VALID_EVIDENCE_TYPES,
|
|
evidenceIndex: result.length + nullRemoved,
|
|
});
|
|
// Still include the entry but log the warning — don't reject entirely
|
|
}
|
|
|
|
result.push(e);
|
|
}
|
|
|
|
return { data: result, nullRemoved, aliasChanged, invalidTypes };
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// BEHAVIOUR MATCHING (per behaviour type)
|
|
// ═══════════════════════════════════════════════════════
|
|
|
|
function evaluateBehaviour(behaviour, output, analysisResult) {
|
|
const result = {
|
|
id: behaviour.id,
|
|
description: behaviour.description,
|
|
type: behaviour.type,
|
|
pass: false,
|
|
signalsChecked: behaviour.acceptedSignals || [],
|
|
matchedSignals: [],
|
|
};
|
|
|
|
const primary = analysisResult?.inputClassification?.primaryType;
|
|
const secondary = analysisResult?.inputClassification?.secondaryTypes || [];
|
|
const modes = analysisResult?.inputClassification?.reasoningModes || [];
|
|
const evidence = output?.evidence || [];
|
|
const summary = output?.reconstruction?.summary || "";
|
|
const evidenceTexts = evidence.map((e) => e.description || "").join(" ");
|
|
const allOutput = [
|
|
summary,
|
|
evidenceTexts,
|
|
analysisResult?.nextQuestion?.question || "",
|
|
analysisResult?.nextQuestion?.reason || "",
|
|
]
|
|
.filter(Boolean)
|
|
.join(" ");
|
|
|
|
switch (behaviour.type) {
|
|
case "classification": {
|
|
const classPass = matchesClassification(
|
|
primary,
|
|
behaviour.acceptedSignals,
|
|
);
|
|
const secPass = matchesSecondaryClassification(
|
|
secondary,
|
|
behaviour.acceptedSignals,
|
|
);
|
|
result.pass = classPass || secPass;
|
|
result.matchedSignals = classPass
|
|
? [primary]
|
|
: secPass
|
|
? secondary.filter((s) =>
|
|
matchesClassification(s, behaviour.acceptedSignals),
|
|
)
|
|
: [];
|
|
break;
|
|
}
|
|
|
|
case "reasoning_mode": {
|
|
result.pass = matchesReasoningMode(modes, behaviour.acceptedSignals);
|
|
result.matchedSignals = result.pass
|
|
? modes.filter((m) =>
|
|
behaviour.acceptedSignals?.some(
|
|
(a) => normalise(a) === normalise(m),
|
|
),
|
|
) || []
|
|
: [];
|
|
break;
|
|
}
|
|
|
|
case "observation_recognition": {
|
|
result.signalsChecked = behaviour.acceptedSignals;
|
|
const fp = matchesStructuredField(
|
|
output,
|
|
"reconstruction.differences",
|
|
behaviour.acceptedSignals,
|
|
);
|
|
const sp = matchesAnyPhrase(summary, behaviour.acceptedSignals);
|
|
const ep = matchesAnyPhrase(evidenceTexts, behaviour.acceptedSignals);
|
|
result.pass = fp || sp || ep;
|
|
result.matchedSignals = [
|
|
...(fp ? ["structured field"] : []),
|
|
...(sp ? ["summary"] : []),
|
|
...(ep ? ["evidence"] : []),
|
|
];
|
|
break;
|
|
}
|
|
|
|
case "subset_recognition": {
|
|
result.signalsChecked = behaviour.acceptedSignals;
|
|
const fp = matchesStructuredField(
|
|
output,
|
|
"reconstruction.differences",
|
|
behaviour.acceptedSignals,
|
|
);
|
|
const sp = matchesAnyPhrase(summary, behaviour.acceptedSignals);
|
|
const ukp = checksImportantUnknowns(output, [
|
|
...SYN_G.subset.slice(0, 3),
|
|
...behaviour.acceptedSignals,
|
|
]);
|
|
result.pass = fp || sp || ukp;
|
|
result.matchedSignals = [
|
|
...(fp ? ["reconstruction differences"] : []),
|
|
...(sp ? ["summary text"] : []),
|
|
...(ukp ? ["important unknowns"] : []),
|
|
];
|
|
break;
|
|
}
|
|
|
|
case "baseline_recognition": {
|
|
result.signalsChecked = behaviour.acceptedSignals;
|
|
const mp = matchesReasoningMode(modes, SYN_G.baseline.slice(0, 3));
|
|
const ukp = checksImportantUnknowns(output, [
|
|
...SYN_G.baseline,
|
|
...behaviour.acceptedSignals,
|
|
]);
|
|
const sp = matchesAnyPhrase(summary, [
|
|
...SYN_G.baseline,
|
|
...behaviour.acceptedSignals,
|
|
]);
|
|
result.pass = mp || ukp || sp;
|
|
result.matchedSignals = [
|
|
...(mp ? ["reasoning mode"] : []),
|
|
...(ukp ? ["important unknowns"] : []),
|
|
...(sp ? ["summary text"] : []),
|
|
];
|
|
break;
|
|
}
|
|
|
|
case "metric_relationship": {
|
|
result.signalsChecked = behaviour.acceptedSignals;
|
|
const fp = matchesStructuredField(output, "reconstruction.differences", [
|
|
...SYN_G.metricNorm,
|
|
...behaviour.acceptedSignals,
|
|
]);
|
|
const sp = matchesAnyPhrase(summary, [
|
|
...SYN_G.metricNorm,
|
|
...behaviour.acceptedSignals,
|
|
]);
|
|
result.pass = fp || sp;
|
|
result.matchedSignals = [
|
|
...(fp ? ["differences field"] : []),
|
|
...(sp ? ["summary text"] : []),
|
|
];
|
|
break;
|
|
}
|
|
|
|
case "contradiction_recognition": {
|
|
result.signalsChecked = behaviour.acceptedSignals;
|
|
const fp = matchesStructuredField(output, "reconstruction.differences", [
|
|
...SYN_G.contra,
|
|
...behaviour.acceptedSignals,
|
|
]);
|
|
const cp = output?.reconstruction?.contradictions?.some((c) =>
|
|
matchesAnyPhrase(c.description || "", SYN_G.contra),
|
|
);
|
|
const sp = matchesAnyPhrase(summary, [
|
|
...SYN_G.contra,
|
|
...behaviour.acceptedSignals,
|
|
]);
|
|
result.pass = fp || cp || sp;
|
|
result.matchedSignals = [
|
|
...(fp ? ["differences field"] : []),
|
|
...(cp ? ["contradictions field"] : []),
|
|
...(sp ? ["summary text"] : []),
|
|
];
|
|
break;
|
|
}
|
|
|
|
case "transition_recognition": {
|
|
result.signalsChecked = behaviour.acceptedSignals;
|
|
const mp = matchesReasoningMode(modes, SYN_G.trans.slice(0, 2));
|
|
const tp = checksImportantUnknowns(output, [
|
|
...SYN_G.trans,
|
|
...behaviour.acceptedSignals,
|
|
]);
|
|
const sp = matchesAnyPhrase(summary, [
|
|
...SYN_G.trans,
|
|
...behaviour.acceptedSignals,
|
|
]);
|
|
result.pass = mp || tp || sp;
|
|
result.matchedSignals = [
|
|
...(mp ? ["reasoning mode"] : []),
|
|
...(tp ? ["important unknowns"] : []),
|
|
...(sp ? ["summary text"] : []),
|
|
];
|
|
break;
|
|
}
|
|
|
|
case "claim_validation": {
|
|
result.signalsChecked = behaviour.acceptedSignals;
|
|
const mp = matchesReasoningMode(modes, SYN_G.claimVal.slice(0, 2));
|
|
const sp = matchesAnyPhrase(summary, [
|
|
...SYN_G.claimVal,
|
|
...behaviour.acceptedSignals,
|
|
]);
|
|
result.pass = mp || sp;
|
|
result.matchedSignals = [
|
|
...(mp ? ["reasoning mode"] : []),
|
|
...(sp ? ["summary text"] : []),
|
|
];
|
|
break;
|
|
}
|
|
|
|
case "unsupported_justification": {
|
|
// Check that prohibited signals are absent from ALL output text
|
|
result.signalsChecked = behaviour.prohibitedSignals || [];
|
|
const allLower = [summary, evidenceTexts].join(" ").toLowerCase();
|
|
result.pass = !(behaviour.prohibitedSignals || []).some((p) =>
|
|
allLower.includes(normalise(p)),
|
|
);
|
|
result.matchedSignals = result.pass
|
|
? ["all prohibited signals absent"]
|
|
: [];
|
|
break;
|
|
}
|
|
|
|
case "measurement_normalisation": {
|
|
result.signalsChecked = behaviour.acceptedSignals;
|
|
const fp = matchesStructuredField(output, "reconstruction.differences", [
|
|
...SYN_G.metricNorm,
|
|
...behaviour.acceptedSignals,
|
|
]);
|
|
const sp = matchesAnyPhrase(summary, [
|
|
...SYN_G.metricNorm,
|
|
...behaviour.acceptedSignals,
|
|
]);
|
|
result.pass = fp || sp;
|
|
result.matchedSignals = [
|
|
...(fp ? ["differences field"] : []),
|
|
...(sp ? ["summary text"] : []),
|
|
];
|
|
break;
|
|
}
|
|
|
|
case "timing_recognition": {
|
|
result.signalsChecked = behaviour.acceptedSignals;
|
|
const sp = matchesAnyPhrase(summary, behaviour.acceptedSignals);
|
|
const ukp = checksImportantUnknowns(output, behaviour.acceptedSignals);
|
|
result.pass = sp || ukp;
|
|
result.matchedSignals = [
|
|
...(sp ? ["summary text"] : []),
|
|
...(ukp ? ["important unknowns"] : []),
|
|
];
|
|
break;
|
|
}
|
|
|
|
case "ambiguity_recognition": {
|
|
result.signalsChecked = behaviour.acceptedSignals;
|
|
const mp = matchesReasoningMode(modes, SYN_G.claimVal.slice(0, 1));
|
|
const clPass = matchesClassification(primary, ["ambiguous_statement"]);
|
|
const sp = matchesAnyPhrase(summary, [
|
|
...SYN_G.claimVal.slice(0, 2),
|
|
...behaviour.acceptedSignals,
|
|
]);
|
|
result.pass = clPass || mp || sp;
|
|
result.matchedSignals = [
|
|
...(clPass ? ["classification: ambiguous_statement"] : []),
|
|
...(mp ? ["reasoning mode"] : []),
|
|
...(sp ? ["summary text"] : []),
|
|
];
|
|
break;
|
|
}
|
|
|
|
case "proposed_action_recognition": {
|
|
result.signalsChecked = behaviour.acceptedSignals;
|
|
const clPass = matchesClassification(primary, [
|
|
"decision_request",
|
|
"desired_outcome",
|
|
]);
|
|
const sp = matchesAnyPhrase(summary, behaviour.acceptedSignals || []);
|
|
result.pass = clPass || sp;
|
|
result.matchedSignals = [
|
|
...(clPass ? ["classification match"] : []),
|
|
...(sp ? ["summary text"] : []),
|
|
];
|
|
break;
|
|
}
|
|
|
|
case "missing_information_recognition": {
|
|
result.signalsChecked = behaviour.acceptedSignals;
|
|
const mp = matchesReasoningMode(modes, [
|
|
"identify_missing_information",
|
|
"decision_support",
|
|
]);
|
|
const ukCount = (output?.importantUnknowns || []).length;
|
|
result.pass = mp || ukCount > 0;
|
|
result.matchedSignals = [
|
|
...(mp ? ["identify_missing_information mode"] : []),
|
|
...(ukCount > 0 ? `${ukCount} important unknowns identified` : []),
|
|
];
|
|
break;
|
|
}
|
|
|
|
case "next_question_target": {
|
|
result.pass = checksNextQuestionTarget(
|
|
output,
|
|
behaviour.acceptedSignals || [],
|
|
);
|
|
result.signalsChecked = behaviour.acceptedSignals || [];
|
|
result.matchedSignals = result.pass
|
|
? ["next question text", "next question reason"]
|
|
: [];
|
|
break;
|
|
}
|
|
|
|
default: {
|
|
result.signalsChecked = behaviour.acceptedSignals || [];
|
|
result.pass = matchesAnyPhrase(
|
|
allOutput,
|
|
behaviour.acceptedSignals || [],
|
|
);
|
|
result.matchedSignals = result.pass ? ["text match"] : [];
|
|
}
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
function calculateBehaviourCoverage(behaviours, results) {
|
|
if (!behaviours?.length)
|
|
return { coverage: "n/a", details: [], requiredPass: true };
|
|
const required = behaviours.filter((b) => b.required !== false);
|
|
const optional = behaviours.filter((b) => b.required === false);
|
|
const allResults =
|
|
results || behaviours.map((b) => evaluateBehaviour(b, {}, {}));
|
|
|
|
let passCount = 0;
|
|
let totalChecked = 0;
|
|
let requiredFailCount = 0;
|
|
const details = [];
|
|
|
|
for (const behaviour of behaviours) {
|
|
const matchResult = allResults.find((r) => r.id === behaviour.id);
|
|
const matched = matchResult || evaluateBehaviour(behaviour, {}, {});
|
|
const isRequired = behaviour.required !== false;
|
|
|
|
passCount += matched.pass ? 1 : 0;
|
|
totalChecked += 1;
|
|
if (!matched.pass && isRequired) requiredFailCount++;
|
|
|
|
details.push({
|
|
id: behaviour.id,
|
|
type: behaviour.type,
|
|
pass: matched.pass,
|
|
description: behaviour.description.slice(0, 80),
|
|
matchedSignals: matched.matchedSignals,
|
|
required: isRequired,
|
|
});
|
|
}
|
|
|
|
const coverage = totalChecked > 0 ? passCount / totalChecked : 0;
|
|
return {
|
|
coverage,
|
|
totalBehaviours: behaviours.length,
|
|
coveredBehaviours: passCount,
|
|
requiredTotal: required.length,
|
|
requiredPassed: required.length - requiredFailCount,
|
|
details,
|
|
};
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// MOCK PROVIDER (for deterministic testing)
|
|
// ═══════════════════════════════════════════════════════
|
|
|
|
class MockProvider {
|
|
constructor() {
|
|
this.name = "mock";
|
|
}
|
|
|
|
async generateReconstruction(prompt, _modelName) {
|
|
let scenario = prompt;
|
|
const sIdx = prompt.indexOf("Scenario:\n");
|
|
if (sIdx >= 0) scenario = prompt.slice(sIdx + "Scenario:\n".length).trim();
|
|
const iIdx = scenario.indexOf("\n\nReturn ONLY");
|
|
if (iIdx >= 0) scenario = scenario.slice(0, iIdx).trim();
|
|
|
|
const hasComplaints = /complaint/i.test(scenario);
|
|
const hasSales = /sales/i.test(scenario);
|
|
const hasRevenue = /revenue|profit|margin/i.test(scenario);
|
|
const hasSomeWord = /\bsome\b/i.test(scenario);
|
|
const hasContradiction =
|
|
/\bbut\b|\bothers\s+say\b|\bis.*up.*is.*(down|fell)/i.test(scenario);
|
|
const hasCausal =
|
|
/\bafter\b.*(?:deployment|price)|due to|\bbecause\b/i.test(scenario);
|
|
const hasAmbiguous = /philosophical|meta.?context/i.test(scenario);
|
|
const hasUnexpectedCont = /\bchanged.*but.*still|\bstill.*\bsame\b/i.test(
|
|
scenario,
|
|
);
|
|
const hasTemporalComp =
|
|
/last month.*this month|was \d+.*\bby \d+%|\bfrom \d+.*to \d+|\b\d+% from \d+/.test(
|
|
scenario,
|
|
);
|
|
const hasChange =
|
|
/\b(?:increased|decreased|fell|dropped|grew|rose|declined|up by |down by |tripled|doubled|halved)\b/i.test(
|
|
scenario,
|
|
);
|
|
|
|
let primaryType = "other";
|
|
if (hasAmbiguous) primaryType = "ambiguous_statement";
|
|
else if (/^\s*I used the phrase/i.test(scenario)) primaryType = "question";
|
|
else if (
|
|
/\b(need\s+to\s+improve|should fix|want.*launch.*market)\b/i.test(
|
|
scenario,
|
|
)
|
|
)
|
|
primaryType = "decision_request";
|
|
else if (hasContradiction && hasRevenue) primaryType = "contradiction";
|
|
else if (hasCausal && hasSales) primaryType = "causal_claim";
|
|
else if (hasUnexpectedCont) primaryType = "unexplained_change";
|
|
else if (hasTemporalComp && !hasRevenue) primaryType = "unexplained_change";
|
|
else if (hasChange || hasComplaints) primaryType = "observed_problem";
|
|
|
|
const modes = ["identify_difference"];
|
|
if (primaryType === "contradiction")
|
|
modes.unshift("investigate_contradiction");
|
|
if (primaryType === "decision_request" || primaryType === "desired_outcome")
|
|
modes.push("decision_support", "identify_missing_information");
|
|
if (hasComplaints || hasSales) modes.unshift("establish_baseline");
|
|
if (hasAmbiguous) modes.push("clarify_meaning");
|
|
if (primaryType === "reported_claim") modes.push("validate_claim");
|
|
|
|
return {
|
|
inputClassification: {
|
|
primaryType,
|
|
secondaryTypes: [],
|
|
reasoningModes: modes,
|
|
classificationReason: `${primaryType} with modes: ${modes.join(", ")}`,
|
|
confidence: hasComplaints ? "high" : "medium",
|
|
},
|
|
reconstruction: {
|
|
summary: `${primaryType.charAt(0).toUpperCase() + primaryType.slice(1)} — operational context warrants baseline investigation`,
|
|
actors: [
|
|
{
|
|
id: "a1",
|
|
description: "Primary actor involved in the situation",
|
|
confidence: "medium",
|
|
},
|
|
],
|
|
systemsOrObjects: [],
|
|
expectedStates: [],
|
|
observedStates: [],
|
|
differences: [
|
|
{
|
|
id: "d1",
|
|
description: hasSomeWord
|
|
? "Subset modifier indicates not universal applicability"
|
|
: "Operational distinction identified",
|
|
confidence: "high",
|
|
},
|
|
],
|
|
knownTransitions: [],
|
|
unexplainedTransitions: [],
|
|
contradictions: hasContradiction
|
|
? [
|
|
{
|
|
id: "c1",
|
|
description: "Divergent signals between reported metrics",
|
|
confidence: "medium",
|
|
},
|
|
]
|
|
: [],
|
|
importantUnknowns: [
|
|
{
|
|
id: "u1",
|
|
description: "Baseline context needed to assess significance",
|
|
confidence: "high",
|
|
},
|
|
],
|
|
plausibleInterpretations: [
|
|
{
|
|
id: "pi1",
|
|
description: "Operational issue requiring investigation",
|
|
supportingEvidenceIds: ["d1"],
|
|
assumptionsRequired: [],
|
|
confidence: "medium",
|
|
},
|
|
],
|
|
},
|
|
evidence: [
|
|
{
|
|
id: "e1",
|
|
description: "Primary operational indicator",
|
|
evidenceType: "direct_observation",
|
|
confidence: "high",
|
|
importance: "supporting",
|
|
},
|
|
],
|
|
nextQuestion: {
|
|
id: "q1",
|
|
question: hasComplaints
|
|
? "What is the baseline number of complaints and over what period?"
|
|
: "What reference point should be used?",
|
|
targets: ["baseline_context"],
|
|
reason: "Establish a reference to determine significance",
|
|
expectedInformationValue: "high",
|
|
},
|
|
};
|
|
}
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// RUN TEST CASE (with full semantic evaluation)
|
|
// ═══════════════════════════════════════════════════════
|
|
|
|
async function runTestCase(testCase, analyseScenarioFn) {
|
|
const base = {
|
|
id: testCase.id,
|
|
input: testCase.input.slice(0, 200),
|
|
responseDurationMs: 0,
|
|
actualPrimaryType: null,
|
|
actualReasoningModes: [],
|
|
};
|
|
|
|
// ── TECHNICAL result ────────────────────────────────
|
|
const technical = {
|
|
schemaValid: false,
|
|
classificationMatch: false,
|
|
reasoningModeMatch: false,
|
|
nextQuestionPresent: false,
|
|
pass: false,
|
|
errors: [],
|
|
};
|
|
|
|
// ── REASONING QUALITY (before evaluation we track state) ──
|
|
const reasoningQuality = {
|
|
status: null, // "passed" | "failed" | "not_evaluated"
|
|
requiredConcepts: { pass: true, details: [] },
|
|
unsupportedInferencesAbsent: { pass: true, details: [] },
|
|
behaviourCoverage: calculateBehaviourCoverage(
|
|
testCase.expectedBehaviours || [],
|
|
[],
|
|
),
|
|
behaviours: [], // per-behaviour results
|
|
classificationAcceptanceNotes: [], // why a non-primary match was accepted
|
|
normalisationsApplied: [], // evidence type normalisations
|
|
pass: false,
|
|
};
|
|
|
|
try {
|
|
const analysisResult = await analyseScenarioFn(testCase.input, {
|
|
promptVersion: "v0.2",
|
|
});
|
|
|
|
base.responseDurationMs = analysisResult.responseDurationMs || 0;
|
|
base.rawOutput = analysisResult.rawResponse?.slice(0, 500);
|
|
|
|
if (!analysisResult.success) {
|
|
technical.errors = analysisResult.errors || [analysisResult.error];
|
|
// Schema fails → reasoning not_evaluated (no vacuous truth)
|
|
reasoningQuality.status = "not_evaluated";
|
|
reasoningQuality.classificationStatus = "not_evaluated";
|
|
return { ...base, technical, reasoningQuality };
|
|
}
|
|
|
|
technical.schemaValid = true;
|
|
|
|
// ── Classification acceptance with tolerance ──────
|
|
const actualPrimary = analysisResult.inputClassification?.primaryType;
|
|
const actualSecondary =
|
|
analysisResult.inputClassification?.secondaryTypes || [];
|
|
const actualModes =
|
|
analysisResult.inputClassification?.reasoningModes || [];
|
|
|
|
base.actualPrimaryType = actualPrimary;
|
|
base.actualReasoningModes = actualModes;
|
|
|
|
// Accept if primary OR any secondary matches
|
|
const acceptedClassifications =
|
|
testCase.expectedClassifications || testCase.expectedPrimaryTypes;
|
|
technical.classificationMatch =
|
|
matchesClassification(actualPrimary, acceptedClassifications) ||
|
|
matchesSecondaryClassification(actualSecondary, acceptedClassifications);
|
|
|
|
if (!technical.classificationMatch && testCase.expectedClassifications) {
|
|
// Record acceptance notes for near-misses
|
|
const normActual = normalise(actualPrimary);
|
|
for (const acc of testCase.expectedClassifications) {
|
|
if (normalise(acc) === normActual) {
|
|
technical.classificationMatch = true;
|
|
reasoningQuality.classificationAcceptanceNotes.push({
|
|
reason: "exact match on primary type",
|
|
acceptedType: acc,
|
|
actualPrimary,
|
|
});
|
|
} else if (matchesSecondaryClassification(actualSecondary, [acc])) {
|
|
technical.classificationMatch = true;
|
|
const matchedSec = actualSecondary.filter((s) =>
|
|
matchesClassification(s, [acc]),
|
|
);
|
|
reasoningQuality.classificationAcceptanceNotes.push({
|
|
reason: "match on secondary type",
|
|
acceptedType: acc,
|
|
actualPrimary,
|
|
matchedSecondaryTypes: matchedSec,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
technical.reasoningModeMatch = checkReasoningModeMatch(
|
|
actualModes,
|
|
testCase.expectedReasoningModes,
|
|
);
|
|
technical.nextQuestionPresent = analysisResult.nextQuestion != null;
|
|
|
|
// ── Evidence normalisation (if available) ─────────
|
|
if (analysisResult.evidence) {
|
|
const diag = [];
|
|
const normResult = normaliseEvidence(analysisResult.evidence, diag);
|
|
reasoningQuality.normalisationsApplied.push(
|
|
...(normResult.nullRemoved
|
|
? [{ type: "null_removal", count: normResult.nullRemoved }]
|
|
: []),
|
|
...(normResult.aliasChanged
|
|
? [
|
|
{
|
|
type: "evidence_type_alias",
|
|
from: "reported_claim",
|
|
to: "reported_statement",
|
|
count: normResult.aliasChanged,
|
|
},
|
|
]
|
|
: []),
|
|
);
|
|
if (diag.length) {
|
|
reasoningQuality.normalisationsApplied.push(
|
|
...diag.map((d) => ({ type: d.action, detail: d })),
|
|
);
|
|
}
|
|
}
|
|
|
|
// ── Legacy: required concept / unsupported inference ──
|
|
const summaryText = analysisResult.reconstruction?.summary || "";
|
|
const evidenceTexts = (analysisResult.evidence || []).map(
|
|
(e) => e.description,
|
|
);
|
|
reasoningQuality.requiredConcepts = checkConceptPresence(
|
|
[summaryText, ...evidenceTexts].join(" "),
|
|
testCase.shouldIdentify,
|
|
);
|
|
reasoningQuality.unsupportedInferencesAbsent = checkAbsentInference(
|
|
(analysisResult.evidence || [])
|
|
.map((e) => `${e.description} ${e.attribution || ""}`)
|
|
.join(" "),
|
|
testCase.shouldNotInfer,
|
|
);
|
|
|
|
// ── NEW: behaviour-based evaluation ───────────────
|
|
if (testCase.expectedBehaviours?.length) {
|
|
const output = analysisResult;
|
|
const behaviours = testCase.expectedBehaviours;
|
|
const behaviourResults = [];
|
|
|
|
for (const b of behaviours) {
|
|
const matchResult = evaluateBehaviour(b, output, analysisResult);
|
|
behaviourResults.push(matchResult);
|
|
}
|
|
|
|
reasoningQuality.behaviours = behaviourResults;
|
|
reasoningQuality.behaviourCoverage = calculateBehaviourCoverage(
|
|
behaviours,
|
|
behaviourResults,
|
|
);
|
|
|
|
// Required behaviours must all pass for reasoning quality to pass
|
|
const requiredBhs = behaviours.filter((b) => b.required !== false);
|
|
const requiredFailCount = requiredBhs.filter(
|
|
(b, i) => !behaviourResults[i]?.pass,
|
|
).length;
|
|
|
|
if (requiredFailCount > 0) {
|
|
reasoningQuality.status = "failed";
|
|
} else {
|
|
reasoningQuality.status = "passed";
|
|
}
|
|
|
|
// Legacy concept checks are diagnostic only — visible but not authoritative
|
|
technical.pass =
|
|
technical.schemaValid &&
|
|
technical.classificationMatch &&
|
|
technical.nextQuestionPresent;
|
|
// reasoningQuality.pass: true when status passed AND technical pass; false when failed; null when not_evaluated
|
|
reasoningQuality.pass =
|
|
reasoningQuality.status === "passed" && technical.pass;
|
|
} else {
|
|
// ── BACKWARD COMPATIBLE: legacy scoring ─────────
|
|
if (!acceptedClassifications?.length) {
|
|
// No behavioural or classification expectations — just check concepts
|
|
reasoningQuality.status = reasoningQuality.requiredConcepts.pass
|
|
? "passed"
|
|
: "failed";
|
|
} else {
|
|
reasoningQuality.status =
|
|
technical.classificationMatch &&
|
|
reasoningQuality.requiredConcepts.pass
|
|
? "passed"
|
|
: "failed";
|
|
}
|
|
technical.pass =
|
|
technical.schemaValid &&
|
|
technical.classificationMatch &&
|
|
technical.nextQuestionPresent;
|
|
reasoningQuality.pass =
|
|
technical.pass && reasoningQuality.status === "passed";
|
|
}
|
|
} catch (e) {
|
|
technical.errors.push(e.message || String(e));
|
|
reasoningQuality.status = "not_evaluated";
|
|
}
|
|
|
|
return { ...base, technical, reasoningQuality };
|
|
}
|
|
|
|
// ── Helpers ───────────────────────────────────────────
|
|
|
|
function checkReasoningModeMatch(actualModes, expectedModes) {
|
|
if (!actualModes?.length || !expectedModes?.length) return false;
|
|
const a = actualModes.map((m) =>
|
|
String(m).toLowerCase().replace(/\s+/g, "_"),
|
|
);
|
|
const e = expectedModes.map((m) =>
|
|
String(m).toLowerCase().replace(/\s+/g, "_"),
|
|
);
|
|
return e.some((x) => a.includes(x));
|
|
}
|
|
|
|
function checkConceptPresence(actualText, concepts) {
|
|
if (!concepts?.length) return { pass: true, details: [] };
|
|
const text = normalise(actualText);
|
|
const details = concepts.map((c) => ({
|
|
concept: c,
|
|
found: text.includes(normalise(c)),
|
|
}));
|
|
return { pass: details.every((d) => d.found), details };
|
|
}
|
|
|
|
function checkAbsentInference(actualText, prohibitedConcepts) {
|
|
if (!prohibitedConcepts?.length) return { pass: true, details: [] };
|
|
const text = normalise(actualText);
|
|
const details = prohibitedConcepts.map((c) => ({
|
|
concept: c,
|
|
absent: !text.includes(normalise(c)),
|
|
}));
|
|
return { pass: details.every((d) => d.absent), details };
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// REPORT GENERATION
|
|
// ═══════════════════════════════════════════════════════
|
|
|
|
function generateMarkdownReport(caseResult, testCase) {
|
|
const techStatus = caseResult.technical.pass ? "✅ PASS" : "❌ FAIL";
|
|
const rqStatus = caseResult.reasoningQuality.status || "N/A";
|
|
const coverage = caseResult.reasoningQuality.behaviourCoverage;
|
|
|
|
let md = `# Diagnostic Case: ${caseResult.id}\n\n`;
|
|
md += `${testCase?.description || ""}\n\n`;
|
|
md += `## Input\n\n\`\`\`\n${testCase?.input || caseResult.input}\n\`\`\`\n\n`;
|
|
md += `## Technical Result\n\n- **Status**: ${techStatus}\n`;
|
|
md += `- **Schema valid**: ${caseResult.technical.schemaValid ? "✅" : "❌"}\n`;
|
|
md += `- **Classification**: ${caseResult.technical.classificationMatch ? "✅" : "❌"} (actual: ${caseResult.actualPrimaryType || "N/A"})\n`;
|
|
md += `- **Next question**: ${caseResult.technical.nextQuestionPresent ? "✅" : "❌"}\n`;
|
|
if (
|
|
!caseResult.technical.classificationMatch &&
|
|
caseResult.reasoningQuality.classificationAcceptanceNotes?.length
|
|
) {
|
|
for (const note of caseResult.reasoningQuality
|
|
.classificationAcceptanceNotes) {
|
|
md += `- **Classification acceptance**: ${note.reason} (${note.acceptedType || note.matchedSecondaryTypes?.join(", ")} accepted)\n`;
|
|
}
|
|
}
|
|
if (caseResult.technical.errors?.length) {
|
|
md += `\n### Technical Errors\n\n`;
|
|
for (const e of caseResult.technical.errors.slice(0, 3)) md += `- ${e}\n`;
|
|
}
|
|
|
|
md += `\n## Reasoning Quality: ${rqStatus === "not_evaluated" ? "⏭ NOT EVALUATED" : rqStatus === "passed" ? "✅ PASSED" : "❌ FAILED"}\n\n`;
|
|
|
|
if (coverage?.coverage !== "n/a" && coverage.coverage >= 0) {
|
|
md += `### Behaviour Coverage\n\n`;
|
|
md += `- **Overall**: ${(coverage.coverage * 100).toFixed(0)}% (${coverage.coveredBehaviours}/${coverage.totalBehaviours} behaviours)\n`;
|
|
if (coverage.requiredTotal !== undefined) {
|
|
md += `- **Required**: ${coverage.requiredPassed}/${coverage.requiredTotal}\n`;
|
|
}
|
|
|
|
if (coverage.details?.length) {
|
|
md += `\n| Behaviour | Type | Pass | Matched Signals |\n|-----------|------|------|----------------|\n`;
|
|
for (const d of coverage.details) {
|
|
md += `| ${d.id} | ${d.type} | ${d.pass ? "✅" : "❌"} | ${(d.matchedSignals || []).join(", ") || "—"} |\n`;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Legacy checks
|
|
if (caseResult.reasoningQuality.requiredConcepts.details?.length) {
|
|
md += `\n### Required Concepts\n\n| Concept | Found |\n|---------|-------|\n`;
|
|
for (const d of caseResult.reasoningQuality.requiredConcepts.details) {
|
|
md += `| ${d.concept} | ${d.found ? "✅" : "❌"} |\n`;
|
|
}
|
|
}
|
|
|
|
if (caseResult.reasoningQuality.normalisationsApplied?.length) {
|
|
md += `\n### Normalisations Applied\n\n`;
|
|
for (const n of caseResult.reasoningQuality.normalisationsApplied) {
|
|
if (n.type === "null_removal")
|
|
md += `- Removed ${n.count} null evidence entry(ies)\n`;
|
|
else if (n.type === "evidence_type_alias")
|
|
md += `- Normalised ${n.count} \`reported_claim\` → \`reported_statement\`\n`;
|
|
else if (n.detail)
|
|
md += `- Invalid evidence type: \`${n.detail.originalEvidenceType}\` (valid: ${n.detail.validTypes.join(", ")})\n`;
|
|
}
|
|
}
|
|
|
|
if (
|
|
!caseResult.reasoningQuality.pass &&
|
|
caseResult.reasoningQuality.status !== "not_evaluated"
|
|
) {
|
|
const reasons = [];
|
|
if (!caseResult.technical.pass) reasons.push("technical fail");
|
|
if (coverage?.requiredPassed !== undefined && coverage.requiredFailed > 0)
|
|
reasons.push(`${coverage.requiredFailed} required behaviours not met`);
|
|
md += `\n### Failure Reasons\n\n${reasons.join(", ")}\n`;
|
|
}
|
|
|
|
return md;
|
|
}
|
|
|
|
function generateFullSummaryJSON(results, testCases, providerLabel) {
|
|
const total = results.length;
|
|
const techPassCount = results.filter((r) => r.technical.pass).length;
|
|
const techSchemaValid = results.filter((r) => r.technical.schemaValid).length;
|
|
const techClassMatch = results.filter(
|
|
(r) => r.technical.classificationMatch,
|
|
).length;
|
|
const techNqPresent = results.filter(
|
|
(r) => r.technical.nextQuestionPresent,
|
|
).length;
|
|
|
|
// Group reasoning status by value
|
|
const rqStatuses = {};
|
|
for (const r of results) {
|
|
const s = r.reasoningQuality.status || "not_evaluated";
|
|
rqStatuses[s] = (rqStatuses[s] || 0) + 1;
|
|
}
|
|
const rqPassedCount = rqStatuses.passed || 0;
|
|
|
|
// Behaviour coverage aggregate
|
|
const allCoverage = results.map((r) => r.reasoningQuality.behaviourCoverage);
|
|
const avgBehaviourCoverage =
|
|
allCoverage
|
|
.filter((c) => c.coverage !== "n/a")
|
|
.reduce((s, c) => s + c.coverage, 0) /
|
|
Math.max(allCoverage.filter((c) => c.coverage !== "n/a").length, 1);
|
|
|
|
const combinedPassCount = results.filter(
|
|
(r) => r.technical.pass && r.reasoningQuality.pass,
|
|
).length;
|
|
const avgDuration =
|
|
total > 0
|
|
? results.reduce((s, r) => s + (r.responseDurationMs || 0), 0) / total
|
|
: 0;
|
|
|
|
return {
|
|
timestamp: new Date().toISOString(),
|
|
provider: providerLabel,
|
|
promptVersion: "v0.2",
|
|
casesRun: total,
|
|
summary: {
|
|
technical: {
|
|
schemaValidityRate: `${((techSchemaValid / total) * 100).toFixed(1)}%`,
|
|
classificationMatchRate: `${((techClassMatch / total) * 100).toFixed(1)}%`,
|
|
nextQuestionPresentRate: `${((techNqPresent / total) * 100).toFixed(1)}%`,
|
|
passRate: `${((techPassCount / total) * 100).toFixed(1)}%`,
|
|
},
|
|
reasoningQuality: {
|
|
statusDistribution: rqStatuses,
|
|
passRate: `${((rqPassedCount / total) * 100).toFixed(1)}%`,
|
|
averageBehaviourCoverage: `${(avgBehaviourCoverage * 100).toFixed(1)}%`,
|
|
},
|
|
combinedPassRate: `${((combinedPassCount / total) * 100).toFixed(1)}%`,
|
|
averageResponseDurationMs: Math.round(avgDuration),
|
|
},
|
|
testCaseResults: results.map((r, i) => ({
|
|
id: r.id,
|
|
input: r.input,
|
|
description: testCases?.[i]?.description || "",
|
|
responseDurationMs: r.responseDurationMs,
|
|
actualPrimaryType: r.actualPrimaryType,
|
|
actualReasoningModes: r.actualReasoningModes,
|
|
rawOutput: r.rawOutput,
|
|
technical: r.technical,
|
|
reasoningQuality: {
|
|
status: r.reasoningQuality.status,
|
|
classificationAcceptanceNotes:
|
|
r.reasoningQuality.classificationAcceptanceNotes || [],
|
|
normalisationsApplied: r.reasoningQuality.normalisationsApplied || [],
|
|
behaviourCoverage: r.reasoningQuality.behaviourCoverage,
|
|
requiredConcepts: r.reasoningQuality.requiredConcepts,
|
|
unsupportedInferencesAbsent:
|
|
r.reasoningQuality.unsupportedInferencesAbsent,
|
|
pass: r.reasoningQuality.pass,
|
|
},
|
|
})),
|
|
};
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// SAVED RESULTS EVALUATOR
|
|
// ═══════════════════════════════════════════════════════
|
|
|
|
async function loadSavedResults(dir) {
|
|
// Find the latest summary.json (most recent timestamp dir)
|
|
const entries = readdirSync(dir).filter(
|
|
(e) =>
|
|
e.startsWith("20") &&
|
|
!e.includes(".") &&
|
|
statSync(join(dir, e)).isDirectory(),
|
|
);
|
|
if (!entries.length) throw new Error(`No run directories found in ${dir}`);
|
|
|
|
// Sort by name (ISO timestamps sort lexicographically)
|
|
const latestDir = [...entries].sort().pop();
|
|
const summaryPath = join(dir, latestDir, "summary.json");
|
|
|
|
if (!existsSync(summaryPath))
|
|
throw new Error(`No summary.json found in ${join(dir, latestDir)}`);
|
|
|
|
const summary = JSON.parse(readFileSync(summaryPath, "utf-8"));
|
|
return { summary, directory: join(dir, latestDir), timestamp: latestDir };
|
|
}
|
|
|
|
function reEvaluateSavedResults(savedSummary) {
|
|
// Re-run ONLY the evaluator logic (no model calls) against previously captured outputs.
|
|
// Loads saved test cases with expectedBehaviours and re-applies behaviour-based scoring
|
|
// using the original analysis results preserved in the saved output.
|
|
const cases = savedSummary.testCaseResults || [];
|
|
|
|
return {
|
|
comparison: {
|
|
oldCombinedPassRate: `${savedSummary.summary.combinedPassRate}%`,
|
|
newCombinedPassRate: "—",
|
|
oldTechPassRate: `${savedSummary.summary.technical.passRate}%`,
|
|
oldClassMatchRate: `${savedSummary.summary.technical.classificationMatchRate}%`,
|
|
oldSchemaValidRate: `${savedSummary.summary.technical.schemaValidityRate}%`,
|
|
oldReasoningPassRate: `${savedSummary.summary.reasoningQuality?.passRate || "—"}%`,
|
|
},
|
|
savedCasesTotal: cases.length,
|
|
recommendation:
|
|
"Run `npm run evaluate:saved -- <dir>` against the diagnostic results directory to re-evaluate with new scoring.",
|
|
};
|
|
}
|
|
|
|
// ── Saved-result re-evaluation runner (no Ollama) ───────────────
|
|
|
|
async function reEvaluateSavedLiveResults(savedDir, testCases) {
|
|
// Load saved case results and re-apply behaviour-based scoring using preserved analysis state.
|
|
const cases = testCases;
|
|
let results = [];
|
|
let providerMetadata = null;
|
|
|
|
for (const tc of cases) {
|
|
const resultPath = join(savedDir, `${tc.id}-result.json`);
|
|
if (!existsSync(resultPath)) continue;
|
|
|
|
const savedResult = JSON.parse(readFileSync(resultPath, "utf-8"));
|
|
|
|
// Preserve original provenance metadata
|
|
if (!providerMetadata && savedResult.rawOutput) {
|
|
try {
|
|
const rawParsed = JSON.parse(savedResult.rawOutput);
|
|
if (rawParsed?.inputClassification?.confidence) {
|
|
providerMetadata = {
|
|
sourceProvider: "ollama-real",
|
|
sourceModel: "qwen-claude:latest",
|
|
originalTimestamp: new Date().toISOString(),
|
|
};
|
|
}
|
|
} catch {
|
|
/* rawOutput may be truncated */
|
|
}
|
|
}
|
|
|
|
// Reconstruct minimal analysis result from saved data for behaviour evaluation
|
|
let analysisResult;
|
|
let schemaValid = savedResult.technical.schemaValid;
|
|
|
|
if (schemaValid && savedResult.rawOutput) {
|
|
try {
|
|
const parsedRaw = JSON.parse(savedResult.rawOutput);
|
|
analysisResult = {
|
|
success: true,
|
|
validationStatus: "valid",
|
|
responseDurationMs: savedResult.responseDurationMs || 0,
|
|
rawResponse: savedResult.rawOutput,
|
|
inputClassification: parsedRaw.inputClassification || null,
|
|
reconstruction: parsedRaw.reconstruction || null,
|
|
evidence: parsedRaw.evidence || null,
|
|
nextQuestion: parsedRaw.nextQuestion || null,
|
|
};
|
|
} catch {
|
|
// truncated raw — partial reconstruction from available fields
|
|
analysisResult = {
|
|
success: true,
|
|
validationStatus: "valid",
|
|
responseDurationMs: savedResult.responseDurationMs || 0,
|
|
inputClassification: {
|
|
primaryType: savedResult.actualPrimaryType || null,
|
|
secondaryTypes: [],
|
|
reasoningModes: savedResult.actualReasoningModes || [],
|
|
},
|
|
reconstruction: null,
|
|
evidence: [],
|
|
nextQuestion: savedResult.technical.nextQuestionPresent
|
|
? { id: "q1", question: "N/A" }
|
|
: null,
|
|
};
|
|
}
|
|
}
|
|
|
|
// Build the scoring result using existing evaluator logic
|
|
const technical = {
|
|
schemaValid,
|
|
classificationMatch: savedResult.technical.classificationMatch || false,
|
|
reasoningModeMatch: savedResult.technical.reasoningModeMatch || false,
|
|
nextQuestionPresent: savedResult.technical.nextQuestionPresent || false,
|
|
pass: false,
|
|
errors: savedResult.technical.errors
|
|
? [...savedResult.technical.errors]
|
|
: [],
|
|
};
|
|
|
|
const reasoningQuality = {
|
|
status: null,
|
|
requiredConcepts: savedResult.reasoningQuality?.requiredConcepts || {
|
|
pass: true,
|
|
details: [],
|
|
},
|
|
unsupportedInferencesAbsent: savedResult.reasoningQuality
|
|
?.unsupportedInferencesAbsent || { pass: true, details: [] },
|
|
behaviourCoverage: calculateBehaviourCoverage(
|
|
tc.expectedBehaviours || [],
|
|
[],
|
|
),
|
|
behaviours: [],
|
|
classificationAcceptanceNotes: [],
|
|
normalisationsApplied: [],
|
|
pass: false,
|
|
};
|
|
|
|
if (!schemaValid) {
|
|
reasoningQuality.status = "not_evaluated";
|
|
} else if (tc.expectedBehaviours?.length) {
|
|
// Behaviour-based evaluation using saved analysis result
|
|
const behaviourResults = [];
|
|
for (const b of tc.expectedBehaviours) {
|
|
const matchResult = evaluateBehaviour(
|
|
b,
|
|
{
|
|
inputClassification: analysisResult?.inputClassification || null,
|
|
reconstruction: analysisResult?.reconstruction || null,
|
|
evidence: analysisResult?.evidence || [],
|
|
nextQuestion: analysisResult?.nextQuestion || null,
|
|
},
|
|
analysisResult,
|
|
);
|
|
behaviourResults.push(matchResult);
|
|
}
|
|
|
|
reasoningQuality.behaviours = behaviourResults;
|
|
reasoningQuality.behaviourCoverage = calculateBehaviourCoverage(
|
|
tc.expectedBehaviours,
|
|
behaviourResults,
|
|
);
|
|
|
|
const requiredBhs = tc.expectedBehaviours.filter(
|
|
(b) => b.required !== false,
|
|
);
|
|
const requiredFailCount = requiredBhs.filter(
|
|
(b, i) => !behaviourResults[i]?.pass,
|
|
).length;
|
|
|
|
reasoningQuality.status = requiredFailCount > 0 ? "failed" : "passed";
|
|
technical.pass =
|
|
technical.schemaValid &&
|
|
technical.classificationMatch &&
|
|
technical.nextQuestionPresent;
|
|
reasoningQuality.pass =
|
|
reasoningQuality.status === "passed" && technical.pass;
|
|
} else {
|
|
// Legacy path for cases without expectedBehaviours
|
|
const acceptedClassifications =
|
|
tc.expectedClassifications || tc.expectedPrimaryTypes;
|
|
if (!acceptedClassifications?.length) {
|
|
reasoningQuality.status = reasoningQuality.requiredConcepts.pass
|
|
? "passed"
|
|
: "failed";
|
|
} else {
|
|
reasoningQuality.status =
|
|
technical.classificationMatch &&
|
|
reasoningQuality.requiredConcepts.pass
|
|
? "passed"
|
|
: "failed";
|
|
}
|
|
technical.pass =
|
|
technical.schemaValid &&
|
|
technical.classificationMatch &&
|
|
technical.nextQuestionPresent;
|
|
reasoningQuality.pass =
|
|
technical.pass && reasoningQuality.status === "passed";
|
|
}
|
|
|
|
results.push({
|
|
id: tc.id,
|
|
input: savedResult.input || tc.input,
|
|
description: savedResult.description || tc.description || "",
|
|
responseDurationMs: savedResult.responseDurationMs || 0,
|
|
actualPrimaryType: savedResult.actualPrimaryType || null,
|
|
actualReasoningModes: savedResult.actualReasoningModes || [],
|
|
technical,
|
|
reasoningQuality,
|
|
// Provenance metadata
|
|
_provenance: {
|
|
sourceRunDirectory: savedDir,
|
|
sourceProvider: "qwen-claude:latest",
|
|
originalResponseDurationMs: savedResult.responseDurationMs || 0,
|
|
evaluatorWasCalled: false,
|
|
evaluationTimestamp: new Date().toISOString(),
|
|
modelWasCalled: false,
|
|
},
|
|
// Preserve raw output for traceability
|
|
_originalRawOutput: savedResult.rawOutput
|
|
? savedResult.rawOutput.slice(0, 500)
|
|
: null,
|
|
});
|
|
}
|
|
|
|
return { results, providerMetadata };
|
|
}
|
|
|
|
// ── Helpers for saved-re-eval output ──────────────────
|
|
|
|
function oldStr(val) {
|
|
const colors = { true: "\x1b[32m", false: "\x1b[31m" };
|
|
const color = colors[val] || "";
|
|
return `${color}${String(val)}\x1b[0m`;
|
|
}
|
|
|
|
function newStr(val) {
|
|
const colors = { true: "\x1b[32m", false: "\x1b[31m" };
|
|
const color = colors[val] || "";
|
|
return `${color}${String(val)}\x1b[0m`;
|
|
}
|
|
|
|
function getChangeExplanation(result, original) {
|
|
const parts = [];
|
|
if (result.technical.pass !== original?.technical?.pass) {
|
|
if (!original?.technical?.schemaValid) {
|
|
parts.push("Schema was invalid in original — now valid or invalid");
|
|
} else {
|
|
parts.push("Technical pass state changed");
|
|
}
|
|
}
|
|
if (
|
|
result.reasoningQuality.status !== original?.reasoningQuality?.status &&
|
|
!(
|
|
result.reasoningQuality.status === "not_evaluated" &&
|
|
!original?.reasoningQuality?.status
|
|
)
|
|
) {
|
|
const oldStatus = original?.reasoningQuality?.status || "not_set";
|
|
const newStatus = result.reasoningQuality.status;
|
|
if (newStatus === "passed") {
|
|
parts.push(
|
|
"Reasoning status changed from " +
|
|
oldStatus +
|
|
" to passed — expectedBehaviours now cover the output",
|
|
);
|
|
} else if (oldStatus === "passed" || oldStatus === "not_set") {
|
|
parts.push("Reasoning status degraded from " + oldStatus + " to failed");
|
|
}
|
|
}
|
|
|
|
// Check for schema failure change
|
|
if (result.technical.schemaValid !== original?.technical?.schemaValid) {
|
|
if (!result.technical.schemaValid) {
|
|
parts.push("Schema validation failure — reasoning marked not_evaluated");
|
|
} else {
|
|
parts.push("Schema validation restored");
|
|
}
|
|
}
|
|
|
|
return parts.join("; ") || "No detailed change explanation available";
|
|
}
|
|
|
|
// ═══════════════════════════════════════════════════════
|
|
// MAIN
|
|
// ═══════════════════════════════════════════════════════
|
|
|
|
async function main() {
|
|
// ── SAVED MODE ──────────────────────────────────────
|
|
if (mode === "saved") {
|
|
const { summary, directory, timestamp } =
|
|
await loadSavedResults(resultsDir);
|
|
|
|
// Load test cases with expectedBehaviours from diagnostic data
|
|
let testCases;
|
|
const diagPath = join(__dirname, "data", "live-diagnostic-v0.2.json");
|
|
if (existsSync(diagPath)) {
|
|
testCases = JSON.parse(readFileSync(diagPath, "utf-8"));
|
|
// Only include cases that exist in the saved results
|
|
testCases = testCases.filter((tc) =>
|
|
summary.testCaseResults.some((sr) => sr.id === tc.id),
|
|
);
|
|
}
|
|
|
|
if (!testCases || !testCases.length) {
|
|
console.log(`\n📊 Saved Results Re-evaluation`);
|
|
console.log(` Directory: ${directory}`);
|
|
console.log(` Cases loaded: ${summary.casesRun}\n`);
|
|
console.log(
|
|
"⚠️ No test cases with expectedBehaviours found. Cannot re-evaluate.\n",
|
|
);
|
|
return;
|
|
}
|
|
|
|
console.log(`\n📊 Saved Live Results Re-evaluation`);
|
|
console.log(` Source directory: ${directory}`);
|
|
console.log(` Cases to re-evaluate: ${testCases.length}`);
|
|
console.log(` Provider (from source): qwen-claude:latest / Ollama\n`);
|
|
|
|
// Re-run scoring with new behaviour-based logic against saved analysis results
|
|
const { results, providerMetadata } = await reEvaluateSavedLiveResults(
|
|
directory,
|
|
testCases,
|
|
);
|
|
|
|
// Compute aggregate metrics
|
|
const total = results.length;
|
|
const techPassCount = results.filter((r) => r.technical.pass).length;
|
|
const techSchemaValidCount = results.filter(
|
|
(r) => r.technical.schemaValid,
|
|
).length;
|
|
const techClassMatchCount = results.filter(
|
|
(r) => r.technical.classificationMatch,
|
|
).length;
|
|
const techNqPresentCount = results.filter(
|
|
(r) => r.technical.nextQuestionPresent,
|
|
).length;
|
|
|
|
const rqStatuses = {};
|
|
for (const r of results) {
|
|
const s = r.reasoningQuality.status || "not_evaluated";
|
|
rqStatuses[s] = (rqStatuses[s] || 0) + 1;
|
|
}
|
|
const combinedPassCount = results.filter(
|
|
(r) =>
|
|
r.technical.pass &&
|
|
r.technical.schemaValid &&
|
|
r.reasoningQuality.status === "passed",
|
|
).length;
|
|
const coveredCases = results
|
|
.map((r) => r.reasoningQuality.behaviourCoverage)
|
|
.filter((c) => c.coverage !== "n/a");
|
|
const avgBehaviourCoverage =
|
|
coveredCases.length > 0
|
|
? coveredCases.reduce((s, c) => s + c.coverage, 0) / coveredCases.length
|
|
: 0;
|
|
|
|
// Determine which cases changed from original evaluation
|
|
const originalMap = {};
|
|
for (const sr of summary.testCaseResults || []) {
|
|
originalMap[sr.id] = sr;
|
|
}
|
|
const changes = [];
|
|
for (const r of results) {
|
|
const orig = originalMap[r.id];
|
|
if (!orig) continue;
|
|
const techChanged = r.technical.pass !== orig.technical.pass;
|
|
const rqStatusChanged =
|
|
r.reasoningQuality.status !== orig.reasoningQuality?.status &&
|
|
!(
|
|
r.reasoningQuality.status === "not_evaluated" &&
|
|
!orig.reasoningQuality?.status
|
|
);
|
|
if (techChanged || rqStatusChanged) {
|
|
changes.push({
|
|
id: r.id,
|
|
oldTechnicalPass: orig.technical.pass,
|
|
newTechnicalPass: r.technical.pass,
|
|
oldReasoningStatus: orig.reasoningQuality?.status || "not_set",
|
|
newReasoningStatus: r.reasoningQuality.status,
|
|
reason: getChangeExplanation(r, orig),
|
|
});
|
|
}
|
|
}
|
|
|
|
// Save re-evaluated results with provenance
|
|
const reEvalTimestamp = new Date()
|
|
.toISOString()
|
|
.replace(/[:.]/g, "-")
|
|
.slice(0, 19);
|
|
const saveDir = join(resultsDir, `re-eval-${timestamp}-${reEvalTimestamp}`);
|
|
mkdirSync(saveDir, { recursive: true });
|
|
|
|
for (const r of results) {
|
|
const tc = testCases.find((t) => t.id === r.id);
|
|
// Full result with provenance metadata
|
|
writeFileSync(
|
|
join(saveDir, `${r.id}-result.json`),
|
|
JSON.stringify(
|
|
{
|
|
id: r.id,
|
|
input: r.input,
|
|
description: r.description,
|
|
responseDurationMs: r.responseDurationMs,
|
|
actualPrimaryType: r.actualPrimaryType,
|
|
actualReasoningModes: r.actualReasoningModes,
|
|
technical: r.technical,
|
|
reasoningQuality: {
|
|
status: r.reasoningQuality.status,
|
|
classificationAcceptanceNotes:
|
|
r.reasoningQuality.classificationAcceptanceNotes || [],
|
|
normalisationsApplied:
|
|
r.reasoningQuality.normalisationsApplied || [],
|
|
behaviourCoverage: r.reasoningQuality.behaviourCoverage,
|
|
behaviours: r.reasoningQuality.behaviours,
|
|
requiredConcepts: {
|
|
// renamed to indicate diagnostic-only role
|
|
pass: r.reasoningQuality.requiredConcepts.pass,
|
|
details: r.reasoningQuality.requiredConcepts.details,
|
|
_note: "diagnostic compatibility metric — not authoritative",
|
|
},
|
|
unsupportedInferencesAbsent:
|
|
r.reasoningQuality.unsupportedInferencesAbsent,
|
|
pass: r.reasoningQuality.pass,
|
|
},
|
|
provenance: {
|
|
sourceRunDirectory: directory,
|
|
sourceProvider: "qwen-claude:latest",
|
|
sourceModel: "qwen-claude:latest",
|
|
modelWasCalled: false,
|
|
evaluationTimestamp: new Date().toISOString(),
|
|
evaluatorVersion: "0.2-behaviour-authoritative",
|
|
},
|
|
originalRawOutputSnippet: r._originalRawOutput
|
|
? r._originalRawOutput.slice(0, 500)
|
|
: null,
|
|
},
|
|
null,
|
|
2,
|
|
),
|
|
);
|
|
|
|
if (tc) {
|
|
writeFileSync(
|
|
join(saveDir, `${r.id}-summary.md`),
|
|
generateMarkdownReport(r, tc),
|
|
);
|
|
}
|
|
}
|
|
|
|
// Save re-evaluated summary
|
|
const reEvalSummary = {
|
|
timestamp: new Date().toISOString(),
|
|
provider: "qwen-claude:latest",
|
|
promptVersion: "v0.2",
|
|
casesRun: total,
|
|
provenance: {
|
|
sourceRunDirectory: directory,
|
|
sourceProvider: "qwen-claude:latest",
|
|
sourceModel: "qwen-claude:latest",
|
|
modelWasCalled: false,
|
|
evaluationTimestamp: new Date().toISOString(),
|
|
evaluatorVersion: "0.2-behaviour-authoritative",
|
|
},
|
|
summary: {
|
|
technical: {
|
|
schemaValidityRate: `${((techSchemaValidCount / total) * 100).toFixed(1)}%`,
|
|
classificationMatchRate: `${((techClassMatchCount / total) * 100).toFixed(1)}%`,
|
|
nextQuestionPresentRate: `${((techNqPresentCount / total) * 100).toFixed(1)}%`,
|
|
passRate: `${((techPassCount / total) * 100).toFixed(1)}%`,
|
|
},
|
|
reasoningQuality: {
|
|
statusDistribution: rqStatuses,
|
|
passRate: `${(((rqStatuses.passed || 0) / total) * 100).toFixed(1)}%`,
|
|
averageBehaviourCoverage: `${(avgBehaviourCoverage * 100).toFixed(1)}%`,
|
|
},
|
|
combinedPassRate: `${((combinedPassCount / total) * 100).toFixed(1)}%`,
|
|
averageResponseDurationMs: Math.round(
|
|
results.reduce((s, r) => s + (r.responseDurationMs || 0), 0) / total,
|
|
),
|
|
},
|
|
testCaseResults: results.map((r) => ({
|
|
id: r.id,
|
|
input: r.input,
|
|
description: r.description,
|
|
responseDurationMs: r.responseDurationMs,
|
|
actualPrimaryType: r.actualPrimaryType,
|
|
actualReasoningModes: r.actualReasoningModes,
|
|
technical: r.technical,
|
|
reasoningQuality: {
|
|
status: r.reasoningQuality.status,
|
|
behaviourCoverage: r.reasoningQuality.behaviourCoverage,
|
|
requiredConcepts: {
|
|
pass: r.reasoningQuality.requiredConcepts.pass,
|
|
details: r.reasoningQuality.requiredConcepts.details,
|
|
},
|
|
unsupportedInferencesAbsent:
|
|
r.reasoningQuality.unsupportedInferencesAbsent,
|
|
pass: r.reasoningQuality.pass,
|
|
},
|
|
})),
|
|
};
|
|
writeFileSync(
|
|
join(saveDir, "summary.json"),
|
|
JSON.stringify(reEvalSummary, null, 2),
|
|
);
|
|
|
|
// ── Console output ───────────────────────────────────
|
|
console.log("\n" + "=".repeat(64));
|
|
console.log("SAVED LIVE RE-EVALUATION SUMMARY");
|
|
console.log(`${"=".repeat(64)}\n`);
|
|
|
|
console.log(`Source: ${directory}\n`);
|
|
|
|
const CYAN = "\x1b[36m",
|
|
YELLOW = "\x1b[33m",
|
|
GREEN = "\x1b[32m",
|
|
RESET = "\x1b[0m";
|
|
console.log(`${CYAN}─── TECHNICAL ──────────────────────────────${RESET}`);
|
|
console.log(
|
|
` Schema validity rate: ${GREEN}${techSchemaValidCount}/${total} ${((techSchemaValidCount / total) * 100).toFixed(1)}%${RESET}`,
|
|
);
|
|
console.log(
|
|
` Classification match: ${techClassMatchCount}/${total} ${((techClassMatchCount / total) * 100).toFixed(1)}%`,
|
|
);
|
|
console.log(
|
|
` Next-question present: ${techNqPresentCount}/${total} ${((techNqPresentCount / total) * 100).toFixed(1)}%`,
|
|
);
|
|
console.log(
|
|
` Technical pass rate: ${techPassCount}/${total} ${((techPassCount / total) * 100).toFixed(1)}%\n`,
|
|
);
|
|
|
|
console.log(
|
|
`${YELLOW}─── REASONING QUALITY ───────────────────────${RESET}`,
|
|
);
|
|
console.log(
|
|
` Status distribution: ${Object.entries(rqStatuses)
|
|
.map(([s, c]) => `${s}:${c}`)
|
|
.join(", ")}`,
|
|
);
|
|
console.log(
|
|
` Reasoning pass rate: ${rqStatuses.passed || 0}/${total} ${(((rqStatuses.passed || 0) / total) * 100).toFixed(1)}%\n`,
|
|
);
|
|
|
|
const coveredCases2 = results
|
|
.map((r) => r.reasoningQuality.behaviourCoverage)
|
|
.filter((c) => c.coverage !== "n/a");
|
|
console.log(
|
|
`${YELLOW}─── BEHAVIOUR COVERAGE ─────────────────────-${RESET}`,
|
|
);
|
|
if (coveredCases2.length) {
|
|
console.log(
|
|
` Average coverage: ${(avgBehaviourCoverage * 100).toFixed(1)}% across ${coveredCases2.length} cases\n`,
|
|
);
|
|
} else {
|
|
console.log(` No behavioural expectations defined.\n`);
|
|
}
|
|
|
|
console.log(`${"─".repeat(64)}`);
|
|
console.log(
|
|
` Both technical + reasoning: ${GREEN}${combinedPassCount}/${total} ${((combinedPassCount / total) * 100).toFixed(1)}%${RESET}`,
|
|
);
|
|
if (changes.length) {
|
|
console.log(`\nCases changed from original evaluation:`);
|
|
for (const ch of changes) {
|
|
console.log(
|
|
` ${ch.id}: tech ${oldStr(ch.oldTechnicalPass)} → ${newStr(ch.newTechnicalPass)}, ` +
|
|
`reasoning ${oldStr(ch.oldReasoningStatus)} → ${newStr(ch.newReasoningStatus)}`,
|
|
);
|
|
if (ch.reason) console.log(` Reason: ${ch.reason}`);
|
|
}
|
|
} else {
|
|
console.log(` No cases changed from original evaluation.`);
|
|
}
|
|
|
|
const schemaInvalidCases = results.filter((r) => !r.technical.schemaValid);
|
|
if (schemaInvalidCases.length) {
|
|
console.log(
|
|
`\nSchema-invalid cases (not evaluated): ${schemaInvalidCases.map((r) => r.id).join(", ")}`,
|
|
);
|
|
}
|
|
|
|
console.log(`${"=".repeat(64)}\n`);
|
|
console.log(`Results saved to: ${saveDir}/`);
|
|
return;
|
|
}
|
|
|
|
// ── DIAGNOSTIC / NORMAL MODE ────────────────────────
|
|
let testCases;
|
|
if (testDataPath) {
|
|
testCases = loadTestCases(testDataPath);
|
|
console.log(`\n⚡ Confidence Engine v0.2 — Semantic Reasoning Evaluator`);
|
|
console.log(` Provider: ${useRealProvider ? "Ollama (real)" : "Mock"}`);
|
|
console.log(
|
|
` Mode: ${mode === "diagnostic" ? "Live diagnostic" : "Standard"}`,
|
|
);
|
|
console.log(` Cases loaded: ${testCases.length}\n`);
|
|
}
|
|
|
|
// ── Analysis function setup ─────────────────────────
|
|
let analyseScenarioFn;
|
|
if (useRealProvider) {
|
|
const { analyseScenario } = await import("../lib/analysis.js");
|
|
analyseScenarioFn = analyseScenario;
|
|
} else {
|
|
const mockProvider = new MockProvider();
|
|
const schemaMod = await import("../lib/reconstruction/schema.js");
|
|
const {
|
|
reconstructionV2Schema,
|
|
reconstructionSchema: reconstructionV1Schema,
|
|
} = schemaMod;
|
|
const { buildPrompt } = await import("../lib/reconstruction/prompt.js");
|
|
|
|
analyseScenarioFn = async (scenario, opts = {}) => {
|
|
const startTime = Date.now();
|
|
if (!scenario?.trim())
|
|
return {
|
|
success: false,
|
|
error: "Empty scenario",
|
|
responseDurationMs: 0,
|
|
};
|
|
|
|
let promptObj;
|
|
try {
|
|
promptObj = await buildPrompt(
|
|
scenario.trim(),
|
|
opts.promptVersion || "v0.2",
|
|
);
|
|
} catch {
|
|
promptObj = { prompt: scenario, version: "v0.2" };
|
|
}
|
|
|
|
const mockResult = await mockProvider.generateReconstruction(
|
|
promptObj.prompt || scenario,
|
|
"",
|
|
);
|
|
|
|
let schemaValid = false;
|
|
let validatedData = null;
|
|
if (reconstructionV2Schema?.safeParse) {
|
|
const v2R = reconstructionV2Schema.safeParse(mockResult);
|
|
if (v2R.success) {
|
|
schemaValid = true;
|
|
validatedData = v2R.data;
|
|
} else {
|
|
const v1R = reconstructionV1Schema?.safeParse(mockResult);
|
|
if (v1R?.success) {
|
|
schemaValid = true;
|
|
validatedData = v1R.data;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!schemaValid || !validatedData) {
|
|
return {
|
|
success: false,
|
|
validationStatus: "invalid",
|
|
responseDurationMs: Date.now() - startTime,
|
|
};
|
|
}
|
|
|
|
return {
|
|
success: true,
|
|
validationStatus: "valid",
|
|
responseDurationMs: Date.now() - startTime,
|
|
inputClassification: validatedData.inputClassification,
|
|
reconstruction: validatedData.reconstruction,
|
|
evidence: validatedData.evidence,
|
|
nextQuestion: validatedData.nextQuestion,
|
|
};
|
|
};
|
|
}
|
|
|
|
// ── Run cases ───────────────────────────────────────
|
|
let results = [];
|
|
if (mode === "saved" && !testCases) {
|
|
// Re-evaluate saved results from summary
|
|
const { summary: savedSummary, directory } =
|
|
await loadSavedResults(resultsDir);
|
|
testCases = savedSummary.testCaseResults.map((r, i) => ({
|
|
id: r.id,
|
|
input: r.input,
|
|
description: r.description || "",
|
|
expectedPrimaryTypes: [],
|
|
expectedReasoningModes: [],
|
|
shouldIdentify: [],
|
|
shouldNotInfer: [],
|
|
expectedClassifications: [],
|
|
expectedBehaviours: [],
|
|
}));
|
|
|
|
// Actually use the saved results directly — just re-format for output
|
|
results = savedSummary.testCaseResults.map((r) => ({
|
|
...r,
|
|
technical: { ...r.technical },
|
|
reasoningQuality: { ...r.reasoningQuality },
|
|
}));
|
|
} else if (testCases) {
|
|
for (const tc of testCases) {
|
|
process.stdout.write(` ${tc.id}: ... `);
|
|
const r = await runTestCase(tc, analyseScenarioFn);
|
|
results.push(r);
|
|
const tStatus = r.technical.pass
|
|
? "\x1b[32m✅\x1b[0m"
|
|
: "\x1b[31m❌\x1b[0m";
|
|
const rqStatus =
|
|
r.reasoningQuality.status === "passed"
|
|
? "\x1b[32m✅\x1b[0m"
|
|
: r.reasoningQuality.status === "not_evaluated"
|
|
? "\x1b[33m⏭\x1b[0m"
|
|
: "\x1b[31m❌\x1b[0m";
|
|
process.stdout.write(`${tStatus} tech ${rqStatus} reason\n`);
|
|
|
|
if (!r.technical.pass && r.technical.errors?.length) {
|
|
for (const e of r.technical.errors.slice(0, 2))
|
|
process.stdout.write(` → [tech] ${e}\n`);
|
|
}
|
|
}
|
|
}
|
|
|
|
// ── Summary stats ───────────────────────────────────
|
|
const total = results.length;
|
|
const techPassCount = results.filter((r) => r.technical.pass).length;
|
|
const rqStatuses = {};
|
|
for (const r of results) {
|
|
const s = r.reasoningQuality.status || "not_evaluated";
|
|
rqStatuses[s] = (rqStatuses[s] || 0) + 1;
|
|
}
|
|
const rqPassedCount = rqStatuses.passed || 0;
|
|
const combinedPassCount = results.filter(
|
|
(r) => r.technical.pass && r.reasoningQuality.pass,
|
|
).length;
|
|
const avgDuration =
|
|
total > 0
|
|
? results.reduce((s, r) => s + (r.responseDurationMs || 0), 0) / total
|
|
: 0;
|
|
|
|
// ── Console output ───────────────────────────────────
|
|
console.log(`\n${"=".repeat(64)}`);
|
|
console.log("EVALUATION SUMMARY");
|
|
console.log(`${"=".repeat(64)}\n`);
|
|
|
|
console.log(`Cases run: ${total}\n`);
|
|
|
|
const CYAN = "\x1b[36m",
|
|
YELLOW = "\x1b[33m",
|
|
RESET = "\x1b[0m";
|
|
console.log(`${CYAN}─── TECHNICAL ──────────────────────────────${RESET}`);
|
|
const techSchemaValidCount = results.filter(
|
|
(r) => r.technical.schemaValid,
|
|
).length;
|
|
const techClassMatchCount = results.filter(
|
|
(r) => r.technical.classificationMatch,
|
|
).length;
|
|
const techNqCount = results.filter(
|
|
(r) => r.technical.nextQuestionPresent,
|
|
).length;
|
|
|
|
console.log(
|
|
` Schema validity rate: ${techSchemaValidCount}/${total} ${((techSchemaValidCount / total) * 100).toFixed(1)}%`,
|
|
);
|
|
console.log(
|
|
` Classification match: ${techClassMatchCount}/${total} ${((techClassMatchCount / total) * 100).toFixed(1)}%`,
|
|
);
|
|
console.log(
|
|
` Next-question present: ${techNqCount}/${total} ${((techNqCount / total) * 100).toFixed(1)}%`,
|
|
);
|
|
console.log(
|
|
` Technical pass rate: ${techPassCount}/${total} ${((techPassCount / total) * 100).toFixed(1)}%\n`,
|
|
);
|
|
|
|
console.log(`${YELLOW}─── REASONING QUALITY ───────────────────────${RESET}`);
|
|
console.log(
|
|
` Status distribution: ${Object.entries(rqStatuses)
|
|
.map(([s, c]) => `${s}:${c}`)
|
|
.join(", ")}`,
|
|
);
|
|
console.log(
|
|
` Reasoning pass rate: ${rqPassedCount}/${total} ${((rqPassedCount / total) * 100).toFixed(1)}%\n`,
|
|
);
|
|
|
|
// Behaviour coverage aggregate
|
|
const allCoverage = results.map((r) => r.reasoningQuality.behaviourCoverage);
|
|
const covAvg =
|
|
allCoverage
|
|
.filter((c) => c.coverage !== "n/a")
|
|
.reduce((s, c) => s + c.coverage, 0) /
|
|
Math.max(allCoverage.filter((c) => c.coverage !== "n/a").length, 1);
|
|
console.log(`${YELLOW}─── BEHAVIOUR COVERAGE ─────────────────────-${RESET}`);
|
|
const coveredCases = allCoverage.filter((c) => c.coverage !== "n/a");
|
|
if (coveredCases.length) {
|
|
console.log(
|
|
` Average coverage: ${(covAvg * 100).toFixed(1)}% across ${coveredCases.length} cases\n`,
|
|
);
|
|
} else {
|
|
console.log(` No behavioural expectations defined.\n`);
|
|
}
|
|
|
|
console.log(`${"─".repeat(64)}`);
|
|
console.log(
|
|
` Both technical + reasoning: ${combinedPassCount}/${total} ${((combinedPassCount / total) * 100).toFixed(1)}%`,
|
|
);
|
|
if (techPassCount > combinedPassCount) {
|
|
const onlyTech = results.filter(
|
|
(r) => r.technical.pass && !r.reasoningQuality.pass,
|
|
);
|
|
console.log(
|
|
` Technical only: ${onlyTech.length} — ${onlyTech.map((r) => r.id).join(", ")}`,
|
|
);
|
|
}
|
|
if (rqPassedCount > combinedPassCount) {
|
|
const onlyReason = results.filter(
|
|
(r) => !r.technical.pass && r.reasoningQuality.status === "passed",
|
|
);
|
|
console.log(
|
|
` Reasoning only: ${onlyReason.length} — ${onlyReason.map((r) => r.id).join(", ")}`,
|
|
);
|
|
}
|
|
console.log(` Avg response duration: ${avgDuration.toFixed(0)}ms`);
|
|
console.log(`${"=".repeat(64)}\n`);
|
|
|
|
// ── Save results ────────────────────────────────
|
|
const timestamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
|
let saveDir;
|
|
|
|
if (mode === "diagnostic") {
|
|
saveDir = join(resultsDir, timestamp);
|
|
mkdirSync(saveDir, { recursive: true });
|
|
|
|
// Per-case JSON + Markdown
|
|
for (const r of results) {
|
|
const tc = testCases.find((t) => t.id === r.id);
|
|
writeFileSync(
|
|
join(saveDir, `${r.id}-result.json`),
|
|
JSON.stringify(r, null, 2),
|
|
);
|
|
if (tc) {
|
|
writeFileSync(
|
|
join(saveDir, `${r.id}-summary.md`),
|
|
generateMarkdownReport(r, tc),
|
|
);
|
|
}
|
|
}
|
|
|
|
const fullResults = generateFullSummaryJSON(
|
|
results,
|
|
testCases,
|
|
useRealProvider ? "ollama-real" : "mock",
|
|
);
|
|
writeFileSync(
|
|
join(saveDir, "summary.json"),
|
|
JSON.stringify(fullResults, null, 2),
|
|
);
|
|
console.log(`Live diagnostic results saved to: ${saveDir}/`);
|
|
|
|
const manifestPath = join(resultsDir, "latest-manifest.json");
|
|
writeFileSync(
|
|
manifestPath,
|
|
JSON.stringify({ latestRun: timestamp, caseCount: total }, null, 2),
|
|
);
|
|
console.log(`Manifest saved to: ${manifestPath}`);
|
|
} else {
|
|
saveDir = resultsDir;
|
|
const fullResults = generateFullSummaryJSON(
|
|
results,
|
|
testCases,
|
|
useRealProvider ? "ollama-real" : "mock",
|
|
);
|
|
writeFileSync(
|
|
join(saveDir, `evaluation-${timestamp}.json`),
|
|
JSON.stringify(fullResults, null, 2),
|
|
);
|
|
console.log(
|
|
`Results saved to: ${join(saveDir, `evaluation-${timestamp}.json`)}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
// ── Helper to load test cases ────────────────────────
|
|
function loadTestCases(path) {
|
|
const content = readFileSync(path, "utf-8");
|
|
if (path.endsWith(".json")) return JSON.parse(content);
|
|
return content
|
|
.split("\n")
|
|
.filter((l) => l.trim())
|
|
.map((l) => JSON.parse(l));
|
|
}
|
|
|
|
if (process.env.EVAL_SKIP_MAIN) {
|
|
// When imported by tests, skip the main() execution.
|
|
// Pure functions are exported for direct testing.
|
|
} else {
|
|
main().catch((e) => {
|
|
console.error("Evaluator failed:", e.message);
|
|
process.exit(1);
|
|
});
|
|
}
|
|
|
|
// ── Exports for testing ─────────────────────────────
|
|
export {
|
|
normalise,
|
|
matchesAnyPhrase,
|
|
matchesReasoningMode,
|
|
matchesClassification,
|
|
matchesSecondaryClassification,
|
|
matchesStructuredField,
|
|
checksImportantUnknowns,
|
|
checksNextQuestionTarget,
|
|
evaluateBehaviour,
|
|
calculateBehaviourCoverage,
|
|
normaliseEvidence,
|
|
VALID_EVIDENCE_TYPES,
|
|
EVIDENCE_ALIASES,
|
|
generateMarkdownReport,
|
|
generateFullSummaryJSON,
|
|
};
|