fix: make behaviour evaluation authoritative

Core fix: For cases with expectedBehaviours, reasoningQuality.status is now
set exclusively from behaviour evaluation results (required behaviour pass/fail).
Legacy concept checks remain visible as diagnostic-only metrics and do not
influence the authoritative result.

Key changes:
- Behaviour-based scoring determines reasoning status (passed/failed)
  instead of legacy concept literal matching
- Schema failure correctly forces not_evaluated (no vacuous truth)
- Saved live results re-evaluator preserves provenance metadata
- Classification tolerance map works bidirectionally for interchangeable types
- normalise() treats underscores as word characters, hyphens as spaces

Tests: 74 passing across both evaluator test suites
- tests/evaluator-behaviour-authoritative.test.mjs (47 tests, new)
- tests/evaluator-semantic.test.mjs (27 tests)
This commit is contained in:
2026-08-01 13:39:01 +01:00
parent 93b905df0a
commit a0bcb12792
12 changed files with 4170 additions and 697 deletions
+8 -4
View File
@@ -1,4 +1,8 @@
import { analyseScenario, PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION } from "@/lib/analysis";
import {
analyseScenario,
PROMPT_VERSIONS,
DEFAULT_PROMPT_VERSION,
} from "@/lib/analysis";
export async function POST(request) {
try {
@@ -7,7 +11,7 @@ export async function POST(request) {
if (!body.scenario || typeof body.scenario !== "string") {
return Response.json(
{ error: "Request must include a 'scenario' string field" },
{ status: 400 }
{ status: 400 },
);
}
@@ -22,7 +26,7 @@ export async function POST(request) {
if (!result.success) {
return Response.json(
{ ...result, reconstruction: result.reconstruction || null },
{ status: Number(result.statusCode) || 500 }
{ status: Number(result.statusCode) || 500 },
);
}
@@ -39,7 +43,7 @@ export async function POST(request) {
} catch (e) {
return Response.json(
{ error: e.message || "Unknown server error", responseDurationMs: 0 },
{ status: 500 }
{ status: 500 },
);
}
}