fix: make behaviour evaluation authoritative
Core fix: For cases with expectedBehaviours, reasoningQuality.status is now set exclusively from behaviour evaluation results (required behaviour pass/fail). Legacy concept checks remain visible as diagnostic-only metrics and do not influence the authoritative result. Key changes: - Behaviour-based scoring determines reasoning status (passed/failed) instead of legacy concept literal matching - Schema failure correctly forces not_evaluated (no vacuous truth) - Saved live results re-evaluator preserves provenance metadata - Classification tolerance map works bidirectionally for interchangeable types - normalise() treats underscores as word characters, hyphens as spaces Tests: 74 passing across both evaluator test suites - tests/evaluator-behaviour-authoritative.test.mjs (47 tests, new) - tests/evaluator-semantic.test.mjs (27 tests)
This commit is contained in:
@@ -48,7 +48,8 @@ export default function ScenarioForm() {
|
||||
const hasReconstruction = result?.reconstruction;
|
||||
const hasNextQuestion = result?.nextQuestion;
|
||||
const hasEvidence = result?.evidence && result.evidence.length > 0;
|
||||
const hasMeaningfulContent = hasClassification || hasReconstruction || hasNextQuestion || hasEvidence;
|
||||
const hasMeaningfulContent =
|
||||
hasClassification || hasReconstruction || hasNextQuestion || hasEvidence;
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
@@ -62,7 +63,9 @@ export default function ScenarioForm() {
|
||||
className="w-full rounded-lg border border-gray-300 px-4 py-3 text-sm focus:border-gray-500 focus:outline-none focus:ring-2 focus:ring-gray-400"
|
||||
/>
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs text-gray-400">{scenario.length}/{MAX_LENGTH}</span>
|
||||
<span className="text-xs text-gray-400">
|
||||
{scenario.length}/{MAX_LENGTH}
|
||||
</span>
|
||||
<button
|
||||
type="submit"
|
||||
disabled={status === "loading" || !scenario.trim()}
|
||||
@@ -84,7 +87,8 @@ export default function ScenarioForm() {
|
||||
{/* Show partial content even on validation failure */}
|
||||
{(hasClassification || hasReconstruction) && (
|
||||
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
|
||||
⚠ Partial result — some fields failed validation. Showing what was accepted.
|
||||
⚠ Partial result — some fields failed validation. Showing what was
|
||||
accepted.
|
||||
</div>
|
||||
)}
|
||||
{hasReconstruction && (
|
||||
@@ -106,13 +110,17 @@ export default function ScenarioForm() {
|
||||
)}
|
||||
|
||||
{status === "loading" && (
|
||||
<div className="py-12 text-center text-sm text-gray-400">Waiting for model response...</div>
|
||||
<div className="py-12 text-center text-sm text-gray-400">
|
||||
Waiting for model response...
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Empty state */}
|
||||
{status === "idle" && (
|
||||
<div className="rounded-lg border border-dashed border-gray-300 bg-gray-50 px-6 py-8 text-center">
|
||||
<p className="text-sm text-gray-400">Enter a scenario above and click Analyse to begin.</p>
|
||||
<p className="text-sm text-gray-400">
|
||||
Enter a scenario above and click Analyse to begin.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user