fix: make behaviour evaluation authoritative

Core fix: For cases with expectedBehaviours, reasoningQuality.status is now
set exclusively from behaviour evaluation results (required behaviour pass/fail).
Legacy concept checks remain visible as diagnostic-only metrics and do not
influence the authoritative result.

Key changes:
- Behaviour-based scoring determines reasoning status (passed/failed)
  instead of legacy concept literal matching
- Schema failure correctly forces not_evaluated (no vacuous truth)
- Saved live results re-evaluator preserves provenance metadata
- Classification tolerance map works bidirectionally for interchangeable types
- normalise() treats underscores as word characters, hyphens as spaces

Tests: 74 passing across both evaluator test suites
- tests/evaluator-behaviour-authoritative.test.mjs (47 tests, new)
- tests/evaluator-semantic.test.mjs (27 tests)
This commit is contained in:
2026-08-01 13:39:01 +01:00
parent 93b905df0a
commit a0bcb12792
12 changed files with 4170 additions and 697 deletions
+18 -4
View File
@@ -10,7 +10,9 @@ const ValidationIndicator = ({ status }) => {
invalid: "❌ Validation failed",
};
return (
<div className={`flex items-center gap-2 ${styles[status] || "text-gray-500"}`}>
<div
className={`flex items-center gap-2 ${styles[status] || "text-gray-500"}`}
>
<span className="font-medium">{labels[status] || status}</span>
</div>
);
@@ -29,8 +31,19 @@ export default function DiagnosticsView({ result }) {
{ label: "Model", value: result.modelName || "?" },
{ label: "Provider", value: "Ollama" },
{ label: "Prompt version", value: result.promptVersion || "?" },
{ label: "Duration", value: result.responseDurationMs != null ? `${result.responseDurationMs}ms` : "?" },
{ label: "Validation", value: <ValidationIndicator status={result.validationStatus || "invalid"} /> },
{
label: "Duration",
value:
result.responseDurationMs != null
? `${result.responseDurationMs}ms`
: "?",
},
{
label: "Validation",
value: (
<ValidationIndicator status={result.validationStatus || "invalid"} />
),
},
];
return (
@@ -49,7 +62,8 @@ export default function DiagnosticsView({ result }) {
{result.rawResponse && (
<details className="mt-4">
<summary className="cursor-pointer text-xs text-gray-500 underline hover:text-gray-700">
View raw model response ({(result.rawResponse?.length || 0).toLocaleString()} chars)
View raw model response (
{(result.rawResponse?.length || 0).toLocaleString()} chars)
</summary>
<pre className="mt-2 max-h-60 overflow-auto rounded bg-gray-900 px-3 py-2 text-xs leading-relaxed text-green-400">
{result.rawResponse}
+178 -67
View File
@@ -10,7 +10,9 @@ const confidenceColor = {
};
const ConfidenceBadge = ({ level }) => (
<span className={`inline-block rounded-full border px-2 py-0.5 text-xs font-medium ${confidenceColor[level] || "text-gray-600 bg-gray-100"}`}>
<span
className={`inline-block rounded-full border px-2 py-0.5 text-xs font-medium ${confidenceColor[level] || "text-gray-600 bg-gray-100"}`}
>
{level}
</span>
);
@@ -42,17 +44,27 @@ const importanceLabels = {
function ClassificationDisplay({ classification }) {
if (!classification) return null;
const p = classification.primaryType || classification.primary_type;
const sec = classification.secondaryTypes || classification.secondary_types || [];
const modes = classification.reasoningModes || classification.reasoning_modes || [];
const sec =
classification.secondaryTypes || classification.secondary_types || [];
const modes =
classification.reasoningModes || classification.reasoning_modes || [];
// Normalize camelCase to snake_case for display if needed
const primaryLabel = String(p).replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
const secLabels = sec.map((s) => s.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()));
const modeLabels = modes.map((m) => m.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()));
const primaryLabel = String(p)
.replace(/_/g, " ")
.replace(/\b\w/g, (c) => c.toUpperCase());
const secLabels = sec.map((s) =>
s.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()),
);
const modeLabels = modes.map((m) =>
m.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()),
);
return (
<div className="rounded-lg border border-blue-200 bg-blue-50 p-4">
<h3 className="mb-2 text-sm font-semibold text-blue-700">Input Classification</h3>
<h3 className="mb-2 text-sm font-semibold text-blue-700">
Input Classification
</h3>
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1.5 text-sm">
<dt className="text-blue-500">Primary type</dt>
<dd className="font-medium">{primaryLabel}</dd>
@@ -69,9 +81,14 @@ function ClassificationDisplay({ classification }) {
</>
)}
<dt className="text-blue-500 pt-1">Classification reason</dt>
<dd className="italic">{classification.classificationReason || classification.classification_reason}</dd>
<dd className="italic">
{classification.classificationReason ||
classification.classification_reason}
</dd>
<dt className="text-blue-500 pt-1">Confidence</dt>
<dd><ConfidenceBadge level={classification.confidence} /></dd>
<dd>
<ConfidenceBadge level={classification.confidence} />
</dd>
</dl>
</div>
);
@@ -83,7 +100,9 @@ function SummaryDisplay({ reconstruction }) {
const summary = reconstruction.summary || reconstruction.Summary;
return (
<div className="rounded-lg border border-gray-200 bg-white p-4">
<h3 className="mb-2 text-sm font-semibold text-gray-600">Reconstruction Summary</h3>
<h3 className="mb-2 text-sm font-semibold text-gray-600">
Reconstruction Summary
</h3>
<p className="text-sm leading-relaxed">{summary}</p>
</div>
);
@@ -98,15 +117,26 @@ function ItemList({ title, items, renderExtra }) {
return (
<div className="mb-4 rounded-lg border border-gray-200 bg-white p-4">
<h3 className="mb-2 text-sm font-semibold text-gray-600">{title} ({count})</h3>
<h3 className="mb-2 text-sm font-semibold text-gray-600">
{title} ({count})
</h3>
<ul className="space-y-2">
{itemsArr.map((item, idx) => (
<li key={item.id || `${title}-${idx}`} className="rounded border border-gray-200 bg-white px-3 py-2 text-sm">
<li
key={item.id || `${title}-${idx}`}
className="rounded border border-gray-200 bg-white px-3 py-2 text-sm"
>
<div className="flex items-center gap-2">
{item.id && <span className="font-mono text-xs text-gray-400">#{item.id}</span>}
{item.id && (
<span className="font-mono text-xs text-gray-400">
#{item.id}
</span>
)}
{item.confidence && <ConfidenceBadge level={item.confidence} />}
{item.importance && (
<span className={`inline-block rounded-full border px-2 py-0.5 text-xs font-medium ${importanceColors[item.importance] || "text-gray-600 bg-gray-100"}`}>
<span
className={`inline-block rounded-full border px-2 py-0.5 text-xs font-medium ${importanceColors[item.importance] || "text-gray-600 bg-gray-100"}`}
>
{importanceLabels[item.importance]}
</span>
)}
@@ -123,23 +153,38 @@ function ItemList({ title, items, renderExtra }) {
// ── Plausible interpretations ───────────────────────
function InterpretationsDisplay({ interpretations }) {
if (!interpretations?.length) return null;
const arr = Array.isArray(interpretations) ? interpretations : [interpretations];
const arr = Array.isArray(interpretations)
? interpretations
: [interpretations];
return (
<div className="mb-4 rounded-lg border border-indigo-200 bg-indigo-50 p-4">
<h3 className="mb-2 text-sm font-semibold text-indigo-700">Plausible Interpretations ({arr.length})</h3>
<h3 className="mb-2 text-sm font-semibold text-indigo-700">
Plausible Interpretations ({arr.length})
</h3>
<ul className="space-y-3">
{arr.map((interp, idx) => (
<li key={interp.id || `${idx}`} className="rounded border border-indigo-200 bg-white px-3 py-2.5 text-sm leading-relaxed">
<li
key={interp.id || `${idx}`}
className="rounded border border-indigo-200 bg-white px-3 py-2.5 text-sm leading-relaxed"
>
<div className="flex items-center gap-2 mb-1">
<span className="font-medium text-indigo-600">{interp.description}</span>
{interp.confidence && <ConfidenceBadge level={interp.confidence} />}
<span className="font-medium text-indigo-600">
{interp.description}
</span>
{interp.confidence && (
<ConfidenceBadge level={interp.confidence} />
)}
</div>
{interp.supportingEvidenceIds?.length > 0 && (
<p className="text-xs text-gray-500">Supporting evidence: {interp.supportingEvidenceIds.join(", ")}</p>
<p className="text-xs text-gray-500">
Supporting evidence: {interp.supportingEvidenceIds.join(", ")}
</p>
)}
{interp.assumptionsRequired?.length > 0 && (
<p className="text-xs italic text-gray-500">Requires assumptions: {interp.assumptionsRequired.join("; ")}</p>
<p className="text-xs italic text-gray-500">
Requires assumptions: {interp.assumptionsRequired.join("; ")}
</p>
)}
</li>
))}
@@ -154,22 +199,37 @@ function NextQuestionDisplay({ question }) {
const q = question.question || question.Question;
const targets = question.targets || question.Targets || [];
const reason = question.reason || question.Reason || "";
const value = question.expectedInformationValue || question.expected_information_value || "medium";
const value =
question.expectedInformationValue ||
question.expected_information_value ||
"medium";
const valueLabel = { low: "Low", medium: "Medium", high: "High" }[value] || "Medium";
const valueColor = { low: "bg-yellow-100 text-yellow-800", medium: "bg-blue-100 text-blue-800", high: "bg-green-100 text-green-800" }[value] || "";
const valueLabel =
{ low: "Low", medium: "Medium", high: "High" }[value] || "Medium";
const valueColor =
{
low: "bg-yellow-100 text-yellow-800",
medium: "bg-blue-100 text-blue-800",
high: "bg-green-100 text-green-800",
}[value] || "";
return (
<div className="rounded-lg border-2 border-green-300 bg-green-50 p-5">
<div className="flex items-center gap-2 mb-2">
<h3 className="text-sm font-bold text-green-800">Next Question</h3>
<span className={`rounded-full px-2 py-0.5 text-xs font-medium ${valueColor}`}>{valueLabel} value</span>
<span
className={`rounded-full px-2 py-0.5 text-xs font-medium ${valueColor}`}
>
{valueLabel} value
</span>
</div>
<p className="mb-2 text-base font-medium text-gray-900">{q}</p>
{targets.length > 0 && (
<p className="text-sm text-gray-600">Targets: {targets.join(", ")}</p>
)}
{reason && <p className="text-sm italic text-gray-500">Because: {reason}</p>}
{reason && (
<p className="text-sm italic text-gray-500">Because: {reason}</p>
)}
</div>
);
}
@@ -189,13 +249,24 @@ function EvidenceDisplay({ evidence }) {
return (
<div className="mb-4 rounded-lg border border-gray-200 bg-white p-4">
<h3 className="mb-2 text-sm font-semibold text-gray-600">Supporting Evidence ({arr.length})</h3>
<h3 className="mb-2 text-sm font-semibold text-gray-600">
Supporting Evidence ({arr.length})
</h3>
<ul className="space-y-2">
{arr.map((item, idx) => (
<li key={item.id || `${idx}`} className="rounded border border-gray-200 bg-white px-3 py-2 text-sm leading-relaxed">
<li
key={item.id || `${idx}`}
className="rounded border border-gray-200 bg-white px-3 py-2 text-sm leading-relaxed"
>
<div className="flex items-center gap-2 mb-0.5 flex-wrap">
{item.id && <span className="font-mono text-xs text-gray-400">#{item.id}</span>}
<span className={`inline-block rounded px-1.5 py-0.5 text-[10px] font-medium ${importanceColors[item.importance] || "text-gray-600 bg-gray-100"}`}>
{item.id && (
<span className="font-mono text-xs text-gray-400">
#{item.id}
</span>
)}
<span
className={`inline-block rounded px-1.5 py-0.5 text-[10px] font-medium ${importanceColors[item.importance] || "text-gray-600 bg-gray-100"}`}
>
{importanceLabels[item.importance]}
</span>
<span className="inline-block rounded px-1.5 py-0.5 text-[10px] font-medium bg-gray-100 text-gray-700">
@@ -205,7 +276,9 @@ function EvidenceDisplay({ evidence }) {
</div>
<p className="text-sm">{item.description}</p>
{(item.source || item.attribution) && (
<p className="mt-0.5 text-xs text-gray-400">Source: {item.source || item.attribution}</p>
<p className="mt-0.5 text-xs text-gray-400">
Source: {item.source || item.attribution}
</p>
)}
</li>
))}
@@ -222,7 +295,8 @@ export default function ReconstructionView({ reconstruction, partial }) {
if (partial) {
return (
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-3 text-sm text-yellow-800">
Partial result some fields failed validation. Showing what was accepted.
Partial result some fields failed validation. Showing what was
accepted.
</div>
);
}
@@ -241,60 +315,97 @@ export default function ReconstructionView({ reconstruction, partial }) {
{/* Key differences */}
{data.reconstruction?.differences && (
<ItemList title="Key Differences" items={data.reconstruction.differences} />
<ItemList
title="Key Differences"
items={data.reconstruction.differences}
/>
)}
{/* Unexplained transitions */}
{data.reconstruction?.unexplainedTransitions && data.reconstruction.unexplainedTransitions.length > 0 && (
<ItemList title="Unexplained Transitions" items={data.reconstruction.unexplainedTransitions} renderExtra={(i) => (
i.entity && <p className="mt-1 text-xs text-gray-500">Entity: {i.entity}</p>
)} />
)}
{data.reconstruction?.unexplainedTransitions &&
data.reconstruction.unexplainedTransitions.length > 0 && (
<ItemList
title="Unexplained Transitions"
items={data.reconstruction.unexplainedTransitions}
renderExtra={(i) =>
i.entity && (
<p className="mt-1 text-xs text-gray-500">Entity: {i.entity}</p>
)
}
/>
)}
{/* Contradictions */}
{data.reconstruction?.contradictions && data.reconstruction.contradictions.length > 0 && (
<ItemList title="Contradictions" items={data.reconstruction.contradictions} />
)}
{data.reconstruction?.contradictions &&
data.reconstruction.contradictions.length > 0 && (
<ItemList
title="Contradictions"
items={data.reconstruction.contradictions}
/>
)}
{/* Important unknowns */}
{data.reconstruction?.importantUnknowns && data.reconstruction.importantUnknowns.length > 0 && (
<ItemList title="Important Unknowns" items={data.reconstruction.importantUnknowns} />
)}
{data.reconstruction?.importantUnknowns &&
data.reconstruction.importantUnknowns.length > 0 && (
<ItemList
title="Important Unknowns"
items={data.reconstruction.importantUnknowns}
/>
)}
{/* Plausible interpretations */}
{data.reconstruction?.plausibleInterpretations && data.reconstruction.plausibleInterpretations.length > 0 && (
<InterpretationsDisplay interpretations={data.reconstruction.plausibleInterpretations} />
)}
{data.reconstruction?.plausibleInterpretations &&
data.reconstruction.plausibleInterpretations.length > 0 && (
<InterpretationsDisplay
interpretations={data.reconstruction.plausibleInterpretations}
/>
)}
{/* Secondary reconstruction categories (actors, systems, etc.) */}
{data.reconstruction?.actors && data.reconstruction.actors.length > 0 && (
<ItemList title="Actors" items={data.reconstruction.actors} />
)}
{data.reconstruction?.systemsOrObjects && data.reconstruction.systemsOrObjects.length > 0 && (
<ItemList title="Systems / Objects" items={data.reconstruction.systemsOrObjects} />
)}
{data.reconstruction?.expectedStates && data.reconstruction.expectedStates.length > 0 && (
<ItemList title="Expected States" items={data.reconstruction.expectedStates} />
)}
{data.reconstruction?.observedStates && data.reconstruction.observedStates.length > 0 && (
<ItemList title="Observed States" items={data.reconstruction.observedStates} />
)}
{data.reconstruction?.knownTransitions && data.reconstruction.knownTransitions.length > 0 && (
<ItemList title="Known Transitions" items={data.reconstruction.knownTransitions} renderExtra={(i) => (
<div className="mt-1 text-xs text-gray-500">
{i.entity && <span>Entity: {i.entity} · </span>}
From "{i.previousState}" To "{i.currentState}" ({i.explanationStatus})
</div>
)} />
)}
{data.reconstruction?.systemsOrObjects &&
data.reconstruction.systemsOrObjects.length > 0 && (
<ItemList
title="Systems / Objects"
items={data.reconstruction.systemsOrObjects}
/>
)}
{data.reconstruction?.expectedStates &&
data.reconstruction.expectedStates.length > 0 && (
<ItemList
title="Expected States"
items={data.reconstruction.expectedStates}
/>
)}
{data.reconstruction?.observedStates &&
data.reconstruction.observedStates.length > 0 && (
<ItemList
title="Observed States"
items={data.reconstruction.observedStates}
/>
)}
{data.reconstruction?.knownTransitions &&
data.reconstruction.knownTransitions.length > 0 && (
<ItemList
title="Known Transitions"
items={data.reconstruction.knownTransitions}
renderExtra={(i) => (
<div className="mt-1 text-xs text-gray-500">
{i.entity && <span>Entity: {i.entity} · </span>}
From "{i.previousState}" To "{i.currentState}" (
{i.explanationStatus})
</div>
)}
/>
)}
{/* Next question — prominent */}
<NextQuestionDisplay question={data.nextQuestion} />
{/* Evidence */}
{data.evidence && (
<EvidenceDisplay evidence={data.evidence} />
)}
{data.evidence && <EvidenceDisplay evidence={data.evidence} />}
</div>
);
}
+13 -5
View File
@@ -48,7 +48,8 @@ export default function ScenarioForm() {
const hasReconstruction = result?.reconstruction;
const hasNextQuestion = result?.nextQuestion;
const hasEvidence = result?.evidence && result.evidence.length > 0;
const hasMeaningfulContent = hasClassification || hasReconstruction || hasNextQuestion || hasEvidence;
const hasMeaningfulContent =
hasClassification || hasReconstruction || hasNextQuestion || hasEvidence;
return (
<div className="space-y-6">
@@ -62,7 +63,9 @@ export default function ScenarioForm() {
className="w-full rounded-lg border border-gray-300 px-4 py-3 text-sm focus:border-gray-500 focus:outline-none focus:ring-2 focus:ring-gray-400"
/>
<div className="flex items-center justify-between">
<span className="text-xs text-gray-400">{scenario.length}/{MAX_LENGTH}</span>
<span className="text-xs text-gray-400">
{scenario.length}/{MAX_LENGTH}
</span>
<button
type="submit"
disabled={status === "loading" || !scenario.trim()}
@@ -84,7 +87,8 @@ export default function ScenarioForm() {
{/* Show partial content even on validation failure */}
{(hasClassification || hasReconstruction) && (
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
Partial result some fields failed validation. Showing what was accepted.
Partial result some fields failed validation. Showing what was
accepted.
</div>
)}
{hasReconstruction && (
@@ -106,13 +110,17 @@ export default function ScenarioForm() {
)}
{status === "loading" && (
<div className="py-12 text-center text-sm text-gray-400">Waiting for model response...</div>
<div className="py-12 text-center text-sm text-gray-400">
Waiting for model response...
</div>
)}
{/* Empty state */}
{status === "idle" && (
<div className="rounded-lg border border-dashed border-gray-300 bg-gray-50 px-6 py-8 text-center">
<p className="text-sm text-gray-400">Enter a scenario above and click Analyse to begin.</p>
<p className="text-sm text-gray-400">
Enter a scenario above and click Analyse to begin.
</p>
</div>
)}