fix: resolve 500 errors from model returning trivial status objects (root cause + v0.2 prompt fix)

Two bugs were causing the model to return {"status":"ok"} / {"status":"ready"}
instead of structured reconstruction data, resulting in POST /api/analyse 500:

1. DOUBLE-WRAPPING BUG (lib/llm/provider.js):
   generateReconstruction() called buildPrompt(scenario) on input that was
   already a fully-built prompt string from analyseScenario(). This wrapped the
   v0.1 prompt (~5000+ chars) in another template layer, producing incomprehensible
   output that the model could not parse as structured JSON.
   Fix: Pass scenario through directly (it is ALREADY a built prompt).

2. MISSING JSON SPEC (prompts/reconstruct-v0.2.md):
   The v0.2 prompt template said 'matching the structure exactly' but never
   defined what that structure was. The model invented its own field names
   (input_classification, reasoning_mode, anchors) with snake_case instead of
   camelCase, which failed Zod validation -> 500 errors.
   Fix: Added explicit JSON schema section with exact key names, enum values,
   and nested structure matching the Zod validation layer.

Additionally:
- Refactored route to use analyseScenario from lib/analysis (centralized)
- Added lib/analysis.js with shared analysis logic
- Updated components to display promptVersion and validation errors
- Added lib/reconstruction/prompt.js v0.1/v0.2 versioning
- Added lib/reconstruction/schema.js v0.2 Zod schemas
- Added debug tool scripts, evaluation results, and comparison findings
This commit is contained in:
2026-08-01 08:57:28 +01:00
parent 18ac3f37ec
commit 956fc2e31e
91 changed files with 17691 additions and 280 deletions
+22 -78
View File
@@ -1,16 +1,9 @@
import { getConfig } from "@/lib/config";
import { getProvider } from "@/lib/llm/provider";
import { reconstructionSchema } from "@/lib/reconstruction/schema";
const MAX_SCENARIO_LENGTH = 10000;
import { analyseScenario, PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION } from "@/lib/analysis";
export async function POST(request) {
const startTime = Date.now();
let rawResponse = null;
try {
const body = await request.json();
if (!body.scenario || typeof body.scenario !== "string") {
return Response.json(
{ error: "Request must include a 'scenario' string field" },
@@ -18,83 +11,34 @@ export async function POST(request) {
);
}
const trimmed = body.scenario.trim();
if (trimmed.length === 0) {
// Optional prompt version override
let promptVersion = DEFAULT_PROMPT_VERSION;
if (body.promptVersion && PROMPT_VERSIONS.includes(body.promptVersion)) {
promptVersion = body.promptVersion;
}
const result = await analyseScenario(body.scenario, { promptVersion });
if (!result.success) {
return Response.json(
{ error: "Scenario cannot be empty" },
{ status: 400 }
{ ...result, reconstruction: result.reconstruction || null },
{ status: Number(result.statusCode) || 500 }
);
}
if (trimmed.length > MAX_SCENARIO_LENGTH) {
return Response.json(
{ error: `Scenario must be under ${MAX_SCENARIO_LENGTH} characters` },
{ status: 400 }
);
}
const configResult = getConfig();
if (!configResult.ok) {
return Response.json(
{ error: "Invalid server configuration" },
{ status: 500 }
);
}
const { OLLAMA_BASE_URL, OLLAMA_MODEL } = configResult.config;
const provider = getProvider();
// Attempt parse to capture raw for debugging
let reconstruction;
try {
reconstruction = await provider.generateReconstruction(trimmed, OLLAMA_MODEL);
} catch (e) {
return Response.json(
{
error: e.message || "Unknown server error",
responseDurationMs: Date.now() - startTime,
modelName: OLLAMA_MODEL,
validationStatus: "invalid",
},
{ status: 500 }
);
}
// Try to stringify for rawResponse display (safe even if it's already an object)
try {
rawResponse = JSON.stringify(reconstruction);
} catch {
rawResponse = String(reconstruction).slice(0, 2000);
}
const duration = Date.now() - startTime;
// Validate with Zod schema
const validationResult = reconstructionSchema.safeParse(reconstruction);
if (!validationResult.success) {
return Response.json({
reconstruction: null,
modelName: OLLAMA_MODEL,
responseDurationMs: duration,
validationStatus: "invalid",
rawResponse: rawResponse?.slice(0, 2000),
errors: validationResult.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`),
});
}
return Response.json({
reconstruction: validationResult.data,
modelName: OLLAMA_MODEL,
responseDurationMs: duration,
validationStatus: "valid",
rawResponse: rawResponse?.slice(0, 2000),
inputClassification: result.inputClassification,
reconstruction: result.reconstruction,
evidence: result.evidence,
nextQuestion: result.nextQuestion,
modelName: result.modelName,
responseDurationMs: result.responseDurationMs,
validationStatus: result.validationStatus,
promptVersion: result.promptVersion,
});
} catch (e) {
const duration = Date.now() - startTime;
return Response.json(
{ error: e.message || "Unknown server error", responseDurationMs: duration },
{ error: e.message || "Unknown server error", responseDurationMs: 0 },
{ status: 500 }
);
}