robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: resolve 500 errors from model returning trivial status objects (root cause + v0.2 prompt fix)

Browse Source 
Two bugs were causing the model to return {"status":"ok"} / {"status":"ready"}
instead of structured reconstruction data, resulting in POST /api/analyse 500:

1. DOUBLE-WRAPPING BUG (lib/llm/provider.js):
   generateReconstruction() called buildPrompt(scenario) on input that was
   already a fully-built prompt string from analyseScenario(). This wrapped the
   v0.1 prompt (~5000+ chars) in another template layer, producing incomprehensible
   output that the model could not parse as structured JSON.
   Fix: Pass scenario through directly (it is ALREADY a built prompt).

2. MISSING JSON SPEC (prompts/reconstruct-v0.2.md):
   The v0.2 prompt template said 'matching the structure exactly' but never
   defined what that structure was. The model invented its own field names
   (input_classification, reasoning_mode, anchors) with snake_case instead of
   camelCase, which failed Zod validation -> 500 errors.
   Fix: Added explicit JSON schema section with exact key names, enum values,
   and nested structure matching the Zod validation layer.

Additionally:
- Refactored route to use analyseScenario from lib/analysis (centralized)
- Added lib/analysis.js with shared analysis logic
- Updated components to display promptVersion and validation errors
- Added lib/reconstruction/prompt.js v0.1/v0.2 versioning
- Added lib/reconstruction/schema.js v0.2 Zod schemas
- Added debug tool scripts, evaluation results, and comparison findings
This commit is contained in:
robbond
2026-08-01 08:57:28 +01:00
parent 18ac3f37ec
commit 956fc2e31e
91 changed files with 17691 additions and 280 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/reconstruction/schema.js
+164 -13
View File
@@ -1,38 +1,51 @@
import { z } from "zod";
const confidenceEnum = z.enum(["low", "medium", "high"]);
// ──────────────────────────────────────────────
// Shared enums (v0.1 & v0.2)
// ──────────────────────────────────────────────
const itemSchema = z.object({
export const confidenceEnum = z.enum(["low", "medium", "high"]);
const importanceEnum = z.enum(["incidental", "supporting", "important", "critical"]);
const expectedInfoValueEnum = z.enum(["low", "medium", "high"]);
// ──────────────────────────────────────────────
// v0.1 — extraction-only schema (preserved)
// ──────────────────────────────────────────────
const confidenceEnumV1 = z.enum(["low", "medium", "high"]);
const itemSchemaV1 = z.object({
id: z.string().min(1),
description: z.string().min(1),
confidence: confidenceEnum,
confidence: confidenceEnumV1,
});
export const reconstructionSchema = z.object({
observations: z.array(itemSchema),
observations: z.array(itemSchemaV1),
reportedClaims: z.array(
itemSchema.extend({
itemSchemaV1.extend({
attributedTo: z.union([z.string().min(1), z.null()]).optional().nullable(),
})
),
assumptions: z.array(itemSchema),
entities: z.array(itemSchema),
assumptions: z.array(itemSchemaV1),
entities: z.array(itemSchemaV1),
transitions: z.array(
itemSchema.extend({
itemSchemaV1.extend({
entity: z.string().min(1),
previousState: z.string().min(1),
currentState: z.string().min(1),
explanationStatus: z.string().min(1),
})
),
expectedButMissing: z.array(itemSchema),
presentButUnexpected: z.array(itemSchema),
contradictions: z.array(itemSchema),
openUncertainties: z.array(itemSchema),
expectedButMissing: z.array(itemSchemaV1),
presentButUnexpected: z.array(itemSchemaV1),
contradictions: z.array(itemSchemaV1),
openUncertainties: z.array(itemSchemaV1),
});
// v0.1 analyse response (used internally)
export const analyseResponseSchema = z.object({
reconstruction: reconstructionSchema,
reconstruction: z.union([reconstructionSchema, z.null()]),
modelName: z.string(),
responseDurationMs: z.number(),
validationStatus: z.enum(["valid", "partial", "invalid"]),
@@ -48,6 +61,133 @@ export const healthResponseSchema = z.object({
error: z.string().nullable(),
});
// ──────────────────────────────────────────────
// v0.2 — reasoning classification + reconstruction
// ──────────────────────────────────────────────
export const inputTypes = /** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").INPUT_TYPE_VALUE>} */ (
z.enum([
"observed_problem",
"unexplained_change",
"contradiction",
"decision_request",
"causal_claim",
"reported_claim",
"fault_report",
"ambiguous_statement",
"question",
"desired_outcome",
"insufficient_context",
"other",
])
);
export const reasoningModes = /** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").REASONING_MODE_VALUE>} */ (
z.enum([
"establish_baseline",
"identify_difference",
"reconstruct_transition",
"decompose_aggregate",
"validate_measurement",
"validate_claim",
"investigate_contradiction",
"clarify_meaning",
"decision_support",
"fault_investigation",
"identify_missing_information",
"test_possible_explanations",
"other",
])
);
const evidenceRecordSchema = z.object({
id: z.string().min(1),
description: z.string().min(1),
evidenceType: z.enum(["direct_observation", "reported_statement", "interpretation", "assumption", "inferred_relationship"]),
source: z.string().optional(),
attribution: z.string().nullable().optional(),
confidence: confidenceEnum,
importance: importanceEnum,
});
const reconstructionSchemaV2 = z.object({
summary: z.string().min(1),
actors: z.array(itemSchemaV1),
systemsOrObjects: z.array(itemSchemaV1),
expectedStates: z.array(itemSchemaV1),
observedStates: z.array(itemSchemaV1),
differences: z.array(itemSchemaV1),
knownTransitions: z.array(
itemSchemaV1.extend({
entity: z.string().min(1),
previousState: z.string().min(1),
currentState: z.string().min(1),
explanationStatus: z.string().min(1),
})
),
unexplainedTransitions: z.array(
itemSchemaV1.extend({
entity: z.string().min(1).optional(),
previousState: z.string().min(1).optional(),
currentState: z.string().min(1).optional(),
})
),
contradictions: z.array(itemSchemaV1),
importantUnknowns: z.array(itemSchemaV1),
plausibleInterpretations: z.array(
z.object({
id: z.string().min(1),
description: z.string().min(1),
supportingEvidenceIds: z.array(z.string()),
assumptionsRequired: z.array(z.string()).optional().default([]),
confidence: confidenceEnum,
})
),
});
const inputClassificationSchema = z.object({
primaryType: inputTypes,
secondaryTypes: z.array(inputTypes).optional().default([]),
reasoningModes: z.array(reasoningModes).optional().default([]),
classificationReason: z.string().min(1),
confidence: confidenceEnum,
});
const nextQuestionSchema = z.object({
id: z.string().min(1),
question: z.string().min(1),
targets: z.array(z.string()),
reason: z.string().min(1),
expectedInformationValue: expectedInfoValueEnum,
reasoningMode: reasoningModes.optional().default("other"),
});
// v0.2 complete analysis response (what the model produces)
export const reconstructionV2Schema = z.object({
inputClassification: inputClassificationSchema,
reconstruction: reconstructionSchemaV2,
evidence: z.array(evidenceRecordSchema),
nextQuestion: nextQuestionSchema,
});
// Outer wrapper for API return (includes diagnostics + v0.2 data)
export const analyseResponseV2Schema = z.object({
inputClassification: inputClassificationSchema.optional(),
reconstruction: reconstructionSchemaV2.optional().nullable(),
evidence: z.array(evidenceRecordSchema).optional(),
nextQuestion: nextQuestionSchema.optional(),
modelName: z.string(),
responseDurationMs: z.number(),
validationStatus: z.enum(["valid", "partial", "invalid"]),
rawResponse: z.string().optional(),
errors: z.array(z.string()).optional(),
promptVersion: z.string().optional(),
});
// ──────────────────────────────────────────────
// Parsing helpers
// ──────────────────────────────────────────────
export function parseReconstruction(raw) {
if (typeof raw === "string") {
try {
@@ -58,3 +198,14 @@ export function parseReconstruction(raw) {
}
return reconstructionSchema.parse(raw);
}
export function parseReconstructionV2(raw) {
if (typeof raw === "string") {
try {
raw = JSON.parse(raw);
} catch {
throw new SyntaxError("Model response is not valid JSON");
}
}
return reconstructionV2Schema.parse(raw);
}