fix: make behaviour evaluation authoritative
Core fix: For cases with expectedBehaviours, reasoningQuality.status is now set exclusively from behaviour evaluation results (required behaviour pass/fail). Legacy concept checks remain visible as diagnostic-only metrics and do not influence the authoritative result. Key changes: - Behaviour-based scoring determines reasoning status (passed/failed) instead of legacy concept literal matching - Schema failure correctly forces not_evaluated (no vacuous truth) - Saved live results re-evaluator preserves provenance metadata - Classification tolerance map works bidirectionally for interchangeable types - normalise() treats underscores as word characters, hyphens as spaces Tests: 74 passing across both evaluator test suites - tests/evaluator-behaviour-authoritative.test.mjs (47 tests, new) - tests/evaluator-semantic.test.mjs (27 tests)
This commit is contained in:
@@ -10,7 +10,9 @@ const ValidationIndicator = ({ status }) => {
|
||||
invalid: "❌ Validation failed",
|
||||
};
|
||||
return (
|
||||
<div className={`flex items-center gap-2 ${styles[status] || "text-gray-500"}`}>
|
||||
<div
|
||||
className={`flex items-center gap-2 ${styles[status] || "text-gray-500"}`}
|
||||
>
|
||||
<span className="font-medium">{labels[status] || status}</span>
|
||||
</div>
|
||||
);
|
||||
@@ -29,8 +31,19 @@ export default function DiagnosticsView({ result }) {
|
||||
{ label: "Model", value: result.modelName || "?" },
|
||||
{ label: "Provider", value: "Ollama" },
|
||||
{ label: "Prompt version", value: result.promptVersion || "?" },
|
||||
{ label: "Duration", value: result.responseDurationMs != null ? `${result.responseDurationMs}ms` : "?" },
|
||||
{ label: "Validation", value: <ValidationIndicator status={result.validationStatus || "invalid"} /> },
|
||||
{
|
||||
label: "Duration",
|
||||
value:
|
||||
result.responseDurationMs != null
|
||||
? `${result.responseDurationMs}ms`
|
||||
: "?",
|
||||
},
|
||||
{
|
||||
label: "Validation",
|
||||
value: (
|
||||
<ValidationIndicator status={result.validationStatus || "invalid"} />
|
||||
),
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
@@ -49,7 +62,8 @@ export default function DiagnosticsView({ result }) {
|
||||
{result.rawResponse && (
|
||||
<details className="mt-4">
|
||||
<summary className="cursor-pointer text-xs text-gray-500 underline hover:text-gray-700">
|
||||
View raw model response ({(result.rawResponse?.length || 0).toLocaleString()} chars)
|
||||
View raw model response (
|
||||
{(result.rawResponse?.length || 0).toLocaleString()} chars)
|
||||
</summary>
|
||||
<pre className="mt-2 max-h-60 overflow-auto rounded bg-gray-900 px-3 py-2 text-xs leading-relaxed text-green-400">
|
||||
{result.rawResponse}
|
||||
|
||||
Reference in New Issue
Block a user