Core fix: For cases with expectedBehaviours, reasoningQuality.status is now set exclusively from behaviour evaluation results (required behaviour pass/fail). Legacy concept checks remain visible as diagnostic-only metrics and do not influence the authoritative result. Key changes: - Behaviour-based scoring determines reasoning status (passed/failed) instead of legacy concept literal matching - Schema failure correctly forces not_evaluated (no vacuous truth) - Saved live results re-evaluator preserves provenance metadata - Classification tolerance map works bidirectionally for interchangeable types - normalise() treats underscores as word characters, hyphens as spaces Tests: 74 passing across both evaluator test suites - tests/evaluator-behaviour-authoritative.test.mjs (47 tests, new) - tests/evaluator-semantic.test.mjs (27 tests)
50 lines
1.4 KiB
JavaScript
50 lines
1.4 KiB
JavaScript
import {
|
|
analyseScenario,
|
|
PROMPT_VERSIONS,
|
|
DEFAULT_PROMPT_VERSION,
|
|
} from "@/lib/analysis";
|
|
|
|
export async function POST(request) {
|
|
try {
|
|
const body = await request.json();
|
|
|
|
if (!body.scenario || typeof body.scenario !== "string") {
|
|
return Response.json(
|
|
{ error: "Request must include a 'scenario' string field" },
|
|
{ status: 400 },
|
|
);
|
|
}
|
|
|
|
// Optional prompt version override
|
|
let promptVersion = DEFAULT_PROMPT_VERSION;
|
|
if (body.promptVersion && PROMPT_VERSIONS.includes(body.promptVersion)) {
|
|
promptVersion = body.promptVersion;
|
|
}
|
|
|
|
const result = await analyseScenario(body.scenario, { promptVersion });
|
|
|
|
if (!result.success) {
|
|
return Response.json(
|
|
{ ...result, reconstruction: result.reconstruction || null },
|
|
{ status: Number(result.statusCode) || 500 },
|
|
);
|
|
}
|
|
|
|
return Response.json({
|
|
inputClassification: result.inputClassification,
|
|
reconstruction: result.reconstruction,
|
|
evidence: result.evidence,
|
|
nextQuestion: result.nextQuestion,
|
|
modelName: result.modelName,
|
|
responseDurationMs: result.responseDurationMs,
|
|
validationStatus: result.validationStatus,
|
|
promptVersion: result.promptVersion,
|
|
});
|
|
} catch (e) {
|
|
return Response.json(
|
|
{ error: e.message || "Unknown server error", responseDurationMs: 0 },
|
|
{ status: 500 },
|
|
);
|
|
}
|
|
}
|