fix: make behaviour evaluation authoritative
Core fix: For cases with expectedBehaviours, reasoningQuality.status is now set exclusively from behaviour evaluation results (required behaviour pass/fail). Legacy concept checks remain visible as diagnostic-only metrics and do not influence the authoritative result. Key changes: - Behaviour-based scoring determines reasoning status (passed/failed) instead of legacy concept literal matching - Schema failure correctly forces not_evaluated (no vacuous truth) - Saved live results re-evaluator preserves provenance metadata - Classification tolerance map works bidirectionally for interchangeable types - normalise() treats underscores as word characters, hyphens as spaces Tests: 74 passing across both evaluator test suites - tests/evaluator-behaviour-authoritative.test.mjs (47 tests, new) - tests/evaluator-semantic.test.mjs (27 tests)
This commit is contained in:
@@ -45,7 +45,10 @@ Return ONLY the JSON object. No markdown, no explanation, no preamble.`;
|
||||
/** Load a versioned prompt from disk and substitute {{SCENARIO}} */
|
||||
async function buildV2Prompt(scenario) {
|
||||
try {
|
||||
const content = await fs.readFile(join(PROMPTS_DIR, "reconstruct-v0.2.md"), "utf-8");
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.2.md"),
|
||||
"utf-8",
|
||||
);
|
||||
return content.replace("{{SCENARIO}}", scenario);
|
||||
} catch {
|
||||
// Fall back to v0.1 prompt if v0.2 file is missing
|
||||
@@ -69,6 +72,7 @@ export async function buildPrompt(scenario, version = "v0.2") {
|
||||
break;
|
||||
}
|
||||
|
||||
const strongJsonHint = "\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.";
|
||||
const strongJsonHint =
|
||||
"\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.";
|
||||
return { prompt: prompt + strongJsonHint, version };
|
||||
}
|
||||
|
||||
@@ -5,7 +5,12 @@ import { z } from "zod";
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
export const confidenceEnum = z.enum(["low", "medium", "high"]);
|
||||
const importanceEnum = z.enum(["incidental", "supporting", "important", "critical"]);
|
||||
const importanceEnum = z.enum([
|
||||
"incidental",
|
||||
"supporting",
|
||||
"important",
|
||||
"critical",
|
||||
]);
|
||||
const expectedInfoValueEnum = z.enum(["low", "medium", "high"]);
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
@@ -24,8 +29,11 @@ export const reconstructionSchema = z.object({
|
||||
observations: z.array(itemSchemaV1),
|
||||
reportedClaims: z.array(
|
||||
itemSchemaV1.extend({
|
||||
attributedTo: z.union([z.string().min(1), z.null()]).optional().nullable(),
|
||||
})
|
||||
attributedTo: z
|
||||
.union([z.string().min(1), z.null()])
|
||||
.optional()
|
||||
.nullable(),
|
||||
}),
|
||||
),
|
||||
assumptions: z.array(itemSchemaV1),
|
||||
entities: z.array(itemSchemaV1),
|
||||
@@ -35,7 +43,7 @@ export const reconstructionSchema = z.object({
|
||||
previousState: z.string().min(1),
|
||||
currentState: z.string().min(1),
|
||||
explanationStatus: z.string().min(1),
|
||||
})
|
||||
}),
|
||||
),
|
||||
expectedButMissing: z.array(itemSchemaV1),
|
||||
presentButUnexpected: z.array(itemSchemaV1),
|
||||
@@ -65,45 +73,53 @@ export const healthResponseSchema = z.object({
|
||||
// v0.2 — reasoning classification + reconstruction
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
export const inputTypes = /** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").INPUT_TYPE_VALUE>} */ (
|
||||
z.enum([
|
||||
"observed_problem",
|
||||
"unexplained_change",
|
||||
"contradiction",
|
||||
"decision_request",
|
||||
"causal_claim",
|
||||
"reported_claim",
|
||||
"fault_report",
|
||||
"ambiguous_statement",
|
||||
"question",
|
||||
"desired_outcome",
|
||||
"insufficient_context",
|
||||
"other",
|
||||
])
|
||||
);
|
||||
export const inputTypes =
|
||||
/** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").INPUT_TYPE_VALUE>} */ (
|
||||
z.enum([
|
||||
"observed_problem",
|
||||
"unexplained_change",
|
||||
"contradiction",
|
||||
"decision_request",
|
||||
"causal_claim",
|
||||
"reported_claim",
|
||||
"fault_report",
|
||||
"ambiguous_statement",
|
||||
"question",
|
||||
"desired_outcome",
|
||||
"insufficient_context",
|
||||
"other",
|
||||
])
|
||||
);
|
||||
|
||||
export const reasoningModes = /** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").REASONING_MODE_VALUE>} */ (
|
||||
z.enum([
|
||||
"establish_baseline",
|
||||
"identify_difference",
|
||||
"reconstruct_transition",
|
||||
"decompose_aggregate",
|
||||
"validate_measurement",
|
||||
"validate_claim",
|
||||
"investigate_contradiction",
|
||||
"clarify_meaning",
|
||||
"decision_support",
|
||||
"fault_investigation",
|
||||
"identify_missing_information",
|
||||
"test_possible_explanations",
|
||||
"other",
|
||||
])
|
||||
);
|
||||
export const reasoningModes =
|
||||
/** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").REASONING_MODE_VALUE>} */ (
|
||||
z.enum([
|
||||
"establish_baseline",
|
||||
"identify_difference",
|
||||
"reconstruct_transition",
|
||||
"decompose_aggregate",
|
||||
"validate_measurement",
|
||||
"validate_claim",
|
||||
"investigate_contradiction",
|
||||
"clarify_meaning",
|
||||
"decision_support",
|
||||
"fault_investigation",
|
||||
"identify_missing_information",
|
||||
"test_possible_explanations",
|
||||
"other",
|
||||
])
|
||||
);
|
||||
|
||||
const evidenceRecordSchema = z.object({
|
||||
id: z.string().min(1),
|
||||
description: z.string().min(1),
|
||||
evidenceType: z.enum(["direct_observation", "reported_statement", "interpretation", "assumption", "inferred_relationship"]),
|
||||
evidenceType: z.enum([
|
||||
"direct_observation",
|
||||
"reported_statement",
|
||||
"interpretation",
|
||||
"assumption",
|
||||
"inferred_relationship",
|
||||
]),
|
||||
source: z.string().optional(),
|
||||
attribution: z.string().nullable().optional(),
|
||||
confidence: confidenceEnum,
|
||||
@@ -123,14 +139,14 @@ const reconstructionSchemaV2 = z.object({
|
||||
previousState: z.string().min(1),
|
||||
currentState: z.string().min(1),
|
||||
explanationStatus: z.string().min(1),
|
||||
})
|
||||
}),
|
||||
),
|
||||
unexplainedTransitions: z.array(
|
||||
itemSchemaV1.extend({
|
||||
entity: z.string().min(1).optional(),
|
||||
previousState: z.string().min(1).optional(),
|
||||
currentState: z.string().min(1).optional(),
|
||||
})
|
||||
}),
|
||||
),
|
||||
contradictions: z.array(itemSchemaV1),
|
||||
importantUnknowns: z.array(itemSchemaV1),
|
||||
@@ -141,7 +157,7 @@ const reconstructionSchemaV2 = z.object({
|
||||
supportingEvidenceIds: z.array(z.string()),
|
||||
assumptionsRequired: z.array(z.string()).optional().default([]),
|
||||
confidence: confidenceEnum,
|
||||
})
|
||||
}),
|
||||
),
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user