fix: make behaviour evaluation authoritative

Core fix: For cases with expectedBehaviours, reasoningQuality.status is now
set exclusively from behaviour evaluation results (required behaviour pass/fail).
Legacy concept checks remain visible as diagnostic-only metrics and do not
influence the authoritative result.

Key changes:
- Behaviour-based scoring determines reasoning status (passed/failed)
  instead of legacy concept literal matching
- Schema failure correctly forces not_evaluated (no vacuous truth)
- Saved live results re-evaluator preserves provenance metadata
- Classification tolerance map works bidirectionally for interchangeable types
- normalise() treats underscores as word characters, hyphens as spaces

Tests: 74 passing across both evaluator test suites
- tests/evaluator-behaviour-authoritative.test.mjs (47 tests, new)
- tests/evaluator-semantic.test.mjs (27 tests)
This commit is contained in:
2026-08-01 13:39:01 +01:00
parent 93b905df0a
commit a0bcb12792
12 changed files with 4170 additions and 697 deletions
+6 -2
View File
@@ -45,7 +45,10 @@ Return ONLY the JSON object. No markdown, no explanation, no preamble.`;
/** Load a versioned prompt from disk and substitute {{SCENARIO}} */
async function buildV2Prompt(scenario) {
try {
const content = await fs.readFile(join(PROMPTS_DIR, "reconstruct-v0.2.md"), "utf-8");
const content = await fs.readFile(
join(PROMPTS_DIR, "reconstruct-v0.2.md"),
"utf-8",
);
return content.replace("{{SCENARIO}}", scenario);
} catch {
// Fall back to v0.1 prompt if v0.2 file is missing
@@ -69,6 +72,7 @@ export async function buildPrompt(scenario, version = "v0.2") {
break;
}
const strongJsonHint = "\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.";
const strongJsonHint =
"\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.";
return { prompt: prompt + strongJsonHint, version };
}
+57 -41
View File
@@ -5,7 +5,12 @@ import { z } from "zod";
// ──────────────────────────────────────────────
export const confidenceEnum = z.enum(["low", "medium", "high"]);
const importanceEnum = z.enum(["incidental", "supporting", "important", "critical"]);
const importanceEnum = z.enum([
"incidental",
"supporting",
"important",
"critical",
]);
const expectedInfoValueEnum = z.enum(["low", "medium", "high"]);
// ──────────────────────────────────────────────
@@ -24,8 +29,11 @@ export const reconstructionSchema = z.object({
observations: z.array(itemSchemaV1),
reportedClaims: z.array(
itemSchemaV1.extend({
attributedTo: z.union([z.string().min(1), z.null()]).optional().nullable(),
})
attributedTo: z
.union([z.string().min(1), z.null()])
.optional()
.nullable(),
}),
),
assumptions: z.array(itemSchemaV1),
entities: z.array(itemSchemaV1),
@@ -35,7 +43,7 @@ export const reconstructionSchema = z.object({
previousState: z.string().min(1),
currentState: z.string().min(1),
explanationStatus: z.string().min(1),
})
}),
),
expectedButMissing: z.array(itemSchemaV1),
presentButUnexpected: z.array(itemSchemaV1),
@@ -65,45 +73,53 @@ export const healthResponseSchema = z.object({
// v0.2 — reasoning classification + reconstruction
// ──────────────────────────────────────────────
export const inputTypes = /** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").INPUT_TYPE_VALUE>} */ (
z.enum([
"observed_problem",
"unexplained_change",
"contradiction",
"decision_request",
"causal_claim",
"reported_claim",
"fault_report",
"ambiguous_statement",
"question",
"desired_outcome",
"insufficient_context",
"other",
])
);
export const inputTypes =
/** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").INPUT_TYPE_VALUE>} */ (
z.enum([
"observed_problem",
"unexplained_change",
"contradiction",
"decision_request",
"causal_claim",
"reported_claim",
"fault_report",
"ambiguous_statement",
"question",
"desired_outcome",
"insufficient_context",
"other",
])
);
export const reasoningModes = /** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").REASONING_MODE_VALUE>} */ (
z.enum([
"establish_baseline",
"identify_difference",
"reconstruct_transition",
"decompose_aggregate",
"validate_measurement",
"validate_claim",
"investigate_contradiction",
"clarify_meaning",
"decision_support",
"fault_investigation",
"identify_missing_information",
"test_possible_explanations",
"other",
])
);
export const reasoningModes =
/** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").REASONING_MODE_VALUE>} */ (
z.enum([
"establish_baseline",
"identify_difference",
"reconstruct_transition",
"decompose_aggregate",
"validate_measurement",
"validate_claim",
"investigate_contradiction",
"clarify_meaning",
"decision_support",
"fault_investigation",
"identify_missing_information",
"test_possible_explanations",
"other",
])
);
const evidenceRecordSchema = z.object({
id: z.string().min(1),
description: z.string().min(1),
evidenceType: z.enum(["direct_observation", "reported_statement", "interpretation", "assumption", "inferred_relationship"]),
evidenceType: z.enum([
"direct_observation",
"reported_statement",
"interpretation",
"assumption",
"inferred_relationship",
]),
source: z.string().optional(),
attribution: z.string().nullable().optional(),
confidence: confidenceEnum,
@@ -123,14 +139,14 @@ const reconstructionSchemaV2 = z.object({
previousState: z.string().min(1),
currentState: z.string().min(1),
explanationStatus: z.string().min(1),
})
}),
),
unexplainedTransitions: z.array(
itemSchemaV1.extend({
entity: z.string().min(1).optional(),
previousState: z.string().min(1).optional(),
currentState: z.string().min(1).optional(),
})
}),
),
contradictions: z.array(itemSchemaV1),
importantUnknowns: z.array(itemSchemaV1),
@@ -141,7 +157,7 @@ const reconstructionSchemaV2 = z.object({
supportingEvidenceIds: z.array(z.string()),
assumptionsRequired: z.array(z.string()).optional().default([]),
confidence: confidenceEnum,
})
}),
),
});