fix: make behaviour evaluation authoritative

Core fix: For cases with expectedBehaviours, reasoningQuality.status is now
set exclusively from behaviour evaluation results (required behaviour pass/fail).
Legacy concept checks remain visible as diagnostic-only metrics and do not
influence the authoritative result.

Key changes:
- Behaviour-based scoring determines reasoning status (passed/failed)
  instead of legacy concept literal matching
- Schema failure correctly forces not_evaluated (no vacuous truth)
- Saved live results re-evaluator preserves provenance metadata
- Classification tolerance map works bidirectionally for interchangeable types
- normalise() treats underscores as word characters, hyphens as spaces

Tests: 74 passing across both evaluator test suites
- tests/evaluator-behaviour-authoritative.test.mjs (47 tests, new)
- tests/evaluator-semantic.test.mjs (27 tests)
This commit is contained in:
2026-08-01 13:39:01 +01:00
parent 93b905df0a
commit a0bcb12792
12 changed files with 4170 additions and 697 deletions
+13 -5
View File
@@ -48,7 +48,8 @@ export default function ScenarioForm() {
const hasReconstruction = result?.reconstruction;
const hasNextQuestion = result?.nextQuestion;
const hasEvidence = result?.evidence && result.evidence.length > 0;
const hasMeaningfulContent = hasClassification || hasReconstruction || hasNextQuestion || hasEvidence;
const hasMeaningfulContent =
hasClassification || hasReconstruction || hasNextQuestion || hasEvidence;
return (
<div className="space-y-6">
@@ -62,7 +63,9 @@ export default function ScenarioForm() {
className="w-full rounded-lg border border-gray-300 px-4 py-3 text-sm focus:border-gray-500 focus:outline-none focus:ring-2 focus:ring-gray-400"
/>
<div className="flex items-center justify-between">
<span className="text-xs text-gray-400">{scenario.length}/{MAX_LENGTH}</span>
<span className="text-xs text-gray-400">
{scenario.length}/{MAX_LENGTH}
</span>
<button
type="submit"
disabled={status === "loading" || !scenario.trim()}
@@ -84,7 +87,8 @@ export default function ScenarioForm() {
{/* Show partial content even on validation failure */}
{(hasClassification || hasReconstruction) && (
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
Partial result some fields failed validation. Showing what was accepted.
Partial result some fields failed validation. Showing what was
accepted.
</div>
)}
{hasReconstruction && (
@@ -106,13 +110,17 @@ export default function ScenarioForm() {
)}
{status === "loading" && (
<div className="py-12 text-center text-sm text-gray-400">Waiting for model response...</div>
<div className="py-12 text-center text-sm text-gray-400">
Waiting for model response...
</div>
)}
{/* Empty state */}
{status === "idle" && (
<div className="rounded-lg border border-dashed border-gray-300 bg-gray-50 px-6 py-8 text-center">
<p className="text-sm text-gray-400">Enter a scenario above and click Analyse to begin.</p>
<p className="text-sm text-gray-400">
Enter a scenario above and click Analyse to begin.
</p>
</div>
)}