fix: make behaviour evaluation authoritative
Core fix: For cases with expectedBehaviours, reasoningQuality.status is now set exclusively from behaviour evaluation results (required behaviour pass/fail). Legacy concept checks remain visible as diagnostic-only metrics and do not influence the authoritative result. Key changes: - Behaviour-based scoring determines reasoning status (passed/failed) instead of legacy concept literal matching - Schema failure correctly forces not_evaluated (no vacuous truth) - Saved live results re-evaluator preserves provenance metadata - Classification tolerance map works bidirectionally for interchangeable types - normalise() treats underscores as word characters, hyphens as spaces Tests: 74 passing across both evaluator test suites - tests/evaluator-behaviour-authoritative.test.mjs (47 tests, new) - tests/evaluator-semantic.test.mjs (27 tests)
This commit is contained in:
@@ -1,4 +1,8 @@
|
||||
import { analyseScenario, PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION } from "@/lib/analysis";
|
||||
import {
|
||||
analyseScenario,
|
||||
PROMPT_VERSIONS,
|
||||
DEFAULT_PROMPT_VERSION,
|
||||
} from "@/lib/analysis";
|
||||
|
||||
export async function POST(request) {
|
||||
try {
|
||||
@@ -7,7 +11,7 @@ export async function POST(request) {
|
||||
if (!body.scenario || typeof body.scenario !== "string") {
|
||||
return Response.json(
|
||||
{ error: "Request must include a 'scenario' string field" },
|
||||
{ status: 400 }
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -22,7 +26,7 @@ export async function POST(request) {
|
||||
if (!result.success) {
|
||||
return Response.json(
|
||||
{ ...result, reconstruction: result.reconstruction || null },
|
||||
{ status: Number(result.statusCode) || 500 }
|
||||
{ status: Number(result.statusCode) || 500 },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -39,7 +43,7 @@ export async function POST(request) {
|
||||
} catch (e) {
|
||||
return Response.json(
|
||||
{ error: e.message || "Unknown server error", responseDurationMs: 0 },
|
||||
{ status: 500 }
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user