fix: make behaviour evaluation authoritative

Core fix: For cases with expectedBehaviours, reasoningQuality.status is now
set exclusively from behaviour evaluation results (required behaviour pass/fail).
Legacy concept checks remain visible as diagnostic-only metrics and do not
influence the authoritative result.

Key changes:
- Behaviour-based scoring determines reasoning status (passed/failed)
  instead of legacy concept literal matching
- Schema failure correctly forces not_evaluated (no vacuous truth)
- Saved live results re-evaluator preserves provenance metadata
- Classification tolerance map works bidirectionally for interchangeable types
- normalise() treats underscores as word characters, hyphens as spaces

Tests: 74 passing across both evaluator test suites
- tests/evaluator-behaviour-authoritative.test.mjs (47 tests, new)
- tests/evaluator-semantic.test.mjs (27 tests)
This commit is contained in:
2026-08-01 13:39:01 +01:00
parent 93b905df0a
commit a0bcb12792
12 changed files with 4170 additions and 697 deletions
+6 -2
View File
@@ -45,7 +45,10 @@ Return ONLY the JSON object. No markdown, no explanation, no preamble.`;
/** Load a versioned prompt from disk and substitute {{SCENARIO}} */
async function buildV2Prompt(scenario) {
try {
const content = await fs.readFile(join(PROMPTS_DIR, "reconstruct-v0.2.md"), "utf-8");
const content = await fs.readFile(
join(PROMPTS_DIR, "reconstruct-v0.2.md"),
"utf-8",
);
return content.replace("{{SCENARIO}}", scenario);
} catch {
// Fall back to v0.1 prompt if v0.2 file is missing
@@ -69,6 +72,7 @@ export async function buildPrompt(scenario, version = "v0.2") {
break;
}
const strongJsonHint = "\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.";
const strongJsonHint =
"\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.";
return { prompt: prompt + strongJsonHint, version };
}