fix: make behaviour evaluation authoritative

Core fix: For cases with expectedBehaviours, reasoningQuality.status is now
set exclusively from behaviour evaluation results (required behaviour pass/fail).
Legacy concept checks remain visible as diagnostic-only metrics and do not
influence the authoritative result.

Key changes:
- Behaviour-based scoring determines reasoning status (passed/failed)
  instead of legacy concept literal matching
- Schema failure correctly forces not_evaluated (no vacuous truth)
- Saved live results re-evaluator preserves provenance metadata
- Classification tolerance map works bidirectionally for interchangeable types
- normalise() treats underscores as word characters, hyphens as spaces

Tests: 74 passing across both evaluator test suites
- tests/evaluator-behaviour-authoritative.test.mjs (47 tests, new)
- tests/evaluator-semantic.test.mjs (27 tests)
This commit is contained in:
2026-08-01 13:39:01 +01:00
parent 93b905df0a
commit a0bcb12792
12 changed files with 4170 additions and 697 deletions
+18 -4
View File
@@ -10,7 +10,9 @@ const ValidationIndicator = ({ status }) => {
invalid: "❌ Validation failed",
};
return (
<div className={`flex items-center gap-2 ${styles[status] || "text-gray-500"}`}>
<div
className={`flex items-center gap-2 ${styles[status] || "text-gray-500"}`}
>
<span className="font-medium">{labels[status] || status}</span>
</div>
);
@@ -29,8 +31,19 @@ export default function DiagnosticsView({ result }) {
{ label: "Model", value: result.modelName || "?" },
{ label: "Provider", value: "Ollama" },
{ label: "Prompt version", value: result.promptVersion || "?" },
{ label: "Duration", value: result.responseDurationMs != null ? `${result.responseDurationMs}ms` : "?" },
{ label: "Validation", value: <ValidationIndicator status={result.validationStatus || "invalid"} /> },
{
label: "Duration",
value:
result.responseDurationMs != null
? `${result.responseDurationMs}ms`
: "?",
},
{
label: "Validation",
value: (
<ValidationIndicator status={result.validationStatus || "invalid"} />
),
},
];
return (
@@ -49,7 +62,8 @@ export default function DiagnosticsView({ result }) {
{result.rawResponse && (
<details className="mt-4">
<summary className="cursor-pointer text-xs text-gray-500 underline hover:text-gray-700">
View raw model response ({(result.rawResponse?.length || 0).toLocaleString()} chars)
View raw model response (
{(result.rawResponse?.length || 0).toLocaleString()} chars)
</summary>
<pre className="mt-2 max-h-60 overflow-auto rounded bg-gray-900 px-3 py-2 text-xs leading-relaxed text-green-400">
{result.rawResponse}