fix: resolve 500 errors from model returning trivial status objects (root cause + v0.2 prompt fix)
Two bugs were causing the model to return {"status":"ok"} / {"status":"ready"}
instead of structured reconstruction data, resulting in POST /api/analyse 500:
1. DOUBLE-WRAPPING BUG (lib/llm/provider.js):
generateReconstruction() called buildPrompt(scenario) on input that was
already a fully-built prompt string from analyseScenario(). This wrapped the
v0.1 prompt (~5000+ chars) in another template layer, producing incomprehensible
output that the model could not parse as structured JSON.
Fix: Pass scenario through directly (it is ALREADY a built prompt).
2. MISSING JSON SPEC (prompts/reconstruct-v0.2.md):
The v0.2 prompt template said 'matching the structure exactly' but never
defined what that structure was. The model invented its own field names
(input_classification, reasoning_mode, anchors) with snake_case instead of
camelCase, which failed Zod validation -> 500 errors.
Fix: Added explicit JSON schema section with exact key names, enum values,
and nested structure matching the Zod validation layer.
Additionally:
- Refactored route to use analyseScenario from lib/analysis (centralized)
- Added lib/analysis.js with shared analysis logic
- Updated components to display promptVersion and validation errors
- Added lib/reconstruction/prompt.js v0.1/v0.2 versioning
- Added lib/reconstruction/schema.js v0.2 Zod schemas
- Added debug tool scripts, evaluation results, and comparison findings
This commit is contained in:
+182
@@ -0,0 +1,182 @@
|
||||
/**
|
||||
* Core analysis pipeline — shared by API routes and evaluation harness.
|
||||
* Calls the provider, parses output, validates against Zod schemas (v0.2 first, v0.1 fallback).
|
||||
*/
|
||||
|
||||
import { getConfig } from "../lib/config.js";
|
||||
import { getProvider } from "../lib/llm/provider.js";
|
||||
import { buildPrompt, PROMPT_VERSIONS } from "../lib/reconstruction/prompt.js";
|
||||
import {
|
||||
reconstructionV2Schema,
|
||||
reconstructionSchema as reconstructionV1Schema,
|
||||
} from "../lib/reconstruction/schema.js";
|
||||
|
||||
const MAX_SCENARIO_LENGTH = 10000;
|
||||
const DEFAULT_PROMPT_VERSION = "v0.2";
|
||||
|
||||
/**
|
||||
* Analyse a scenario string through the full pipeline.
|
||||
* @param {string} scenario - The scenario text to analyse
|
||||
* @param {object} [opts]
|
||||
* @param {"v0.1" | "v0.2"} [opts.promptVersion="v0.2"] - Prompt version to use
|
||||
* @returns {Promise<object>} Analysis result with diagnostics
|
||||
*/
|
||||
export async function analyseScenario(scenario, opts = {}) {
|
||||
const startTime = Date.now();
|
||||
|
||||
// ── Input validation ───────────────────────────────
|
||||
if (typeof scenario !== "string") {
|
||||
return buildErrorResponse("Input must be a string", startTime);
|
||||
}
|
||||
|
||||
const trimmed = scenario.trim();
|
||||
if (trimmed.length === 0) {
|
||||
return buildErrorResponse("Scenario cannot be empty", startTime);
|
||||
}
|
||||
if (trimmed.length > MAX_SCENARIO_LENGTH) {
|
||||
return buildErrorResponse(`Scenario must be under ${MAX_SCENARIO_LENGTH} characters`, startTime);
|
||||
}
|
||||
|
||||
// ── Configuration check ────────────────────────────
|
||||
const configResult = getConfig();
|
||||
if (!configResult.ok) {
|
||||
return buildErrorResponse("Invalid server configuration", startTime, "500");
|
||||
}
|
||||
|
||||
const { OLLAMA_BASE_URL: _ignored, OLLAMA_MODEL } = configResult.config;
|
||||
const promptVersion = opts.promptVersion || DEFAULT_PROMPT_VERSION;
|
||||
|
||||
// ── Build prompt ───────────────────────────────────
|
||||
let promptObj;
|
||||
try {
|
||||
promptObj = await buildPrompt(trimmed, promptVersion);
|
||||
} catch (e) {
|
||||
return buildErrorResponse(`Failed to build prompt: ${e.message}`, startTime);
|
||||
}
|
||||
|
||||
// ── Call provider ──────────────────────────────────
|
||||
const provider = getProvider();
|
||||
let rawResponse;
|
||||
try {
|
||||
rawResponse = await provider.generateReconstruction(promptObj.prompt, OLLAMA_MODEL);
|
||||
} catch (e) {
|
||||
return buildErrorResponse(
|
||||
e.message || "Provider error during analysis",
|
||||
Date.now() - startTime
|
||||
);
|
||||
}
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
// Try to capture raw response for diagnostics
|
||||
let rawResponseStr;
|
||||
try {
|
||||
rawResponseStr = JSON.stringify(rawResponse);
|
||||
} catch {
|
||||
rawResponseStr = String(rawResponse).slice(0, 2000);
|
||||
}
|
||||
|
||||
// ── Validate against v0.2 schema (preferred) ──────
|
||||
const resultV2 = tryValidateAgainstSchema(rawResponse, reconstructionV2Schema);
|
||||
if (resultV2.valid) {
|
||||
return buildSuccessResultV2(resultV2.data, OLLAMA_MODEL, duration, promptVersion);
|
||||
}
|
||||
|
||||
// ── Fallback to v0.1 schema ────────────────────────
|
||||
const resultV1 = tryValidateAgainstSchema(rawResponse, reconstructionV1Schema);
|
||||
if (resultV1.valid) {
|
||||
return buildSuccessResultV1(resultV1.data, OLLAMA_MODEL, duration, promptVersion);
|
||||
}
|
||||
|
||||
// ── Neither schema matched — partial failure ───────
|
||||
return buildPartialResult(
|
||||
rawResponseStr?.slice(0, 2000),
|
||||
resultV2.error ?? resultV1.error,
|
||||
OLLAMA_MODEL,
|
||||
duration,
|
||||
promptVersion
|
||||
);
|
||||
}
|
||||
|
||||
/** Attempt validation against a Zod schema */
|
||||
function tryValidateAgainstSchema(data, schema) {
|
||||
if (!schema.safeParse) {
|
||||
return { valid: false, error: new Error("Schema does not support safeParse") };
|
||||
}
|
||||
const result = schema.safeParse(data);
|
||||
return result.success ? { valid: true, data: result.data } : { valid: false, error: result.error };
|
||||
}
|
||||
|
||||
// ── Result builders ──────────────────────────────────
|
||||
|
||||
function buildErrorResponse(message, elapsed, statusCode = 500) {
|
||||
return {
|
||||
success: false,
|
||||
error: message,
|
||||
modelName: null,
|
||||
responseDurationMs: elapsed,
|
||||
validationStatus: "invalid",
|
||||
rawResponse: null,
|
||||
promptVersion: null,
|
||||
statusCode,
|
||||
};
|
||||
}
|
||||
|
||||
function buildSuccessResultV2(data, model, duration, version) {
|
||||
return {
|
||||
success: true,
|
||||
validationStatus: "valid",
|
||||
modelName: model,
|
||||
responseDurationMs: duration,
|
||||
rawResponse: JSON.stringify(data).slice(0, 3000),
|
||||
promptVersion: version,
|
||||
inputClassification: data.inputClassification,
|
||||
reconstruction: data.reconstruction,
|
||||
evidence: data.evidence,
|
||||
nextQuestion: data.nextQuestion,
|
||||
errors: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function buildSuccessResultV1(data, model, duration, version) {
|
||||
return {
|
||||
success: true,
|
||||
validationStatus: "valid",
|
||||
modelName: model,
|
||||
responseDurationMs: duration,
|
||||
rawResponse: JSON.stringify(data).slice(0, 3000),
|
||||
promptVersion: version,
|
||||
inputClassification: null,
|
||||
reconstruction: data,
|
||||
evidence: undefined,
|
||||
nextQuestion: undefined,
|
||||
errors: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function buildPartialResult(rawResp, error, model, duration, version) {
|
||||
let errors = [];
|
||||
if (error && typeof error.flatten === "function") {
|
||||
errors = error.flatten().fieldErrors
|
||||
? Object.entries(error.flatten().fieldErrors).flatMap(([k, v]) => [`${k}: ${v.join(", ")}`])
|
||||
: [String(error)];
|
||||
} else if (error) {
|
||||
errors = [String(error).slice(0, 500)];
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
validationStatus: "invalid",
|
||||
modelName: model,
|
||||
responseDurationMs: duration,
|
||||
rawResponse: rawResp?.slice(0, 2000),
|
||||
promptVersion: version,
|
||||
inputClassification: null,
|
||||
reconstruction: null,
|
||||
evidence: undefined,
|
||||
nextQuestion: undefined,
|
||||
errors,
|
||||
};
|
||||
}
|
||||
|
||||
export { PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION };
|
||||
Reference in New Issue
Block a user