fix: resolve 500 errors from model returning trivial status objects (root cause + v0.2 prompt fix)
Two bugs were causing the model to return {"status":"ok"} / {"status":"ready"}
instead of structured reconstruction data, resulting in POST /api/analyse 500:
1. DOUBLE-WRAPPING BUG (lib/llm/provider.js):
generateReconstruction() called buildPrompt(scenario) on input that was
already a fully-built prompt string from analyseScenario(). This wrapped the
v0.1 prompt (~5000+ chars) in another template layer, producing incomprehensible
output that the model could not parse as structured JSON.
Fix: Pass scenario through directly (it is ALREADY a built prompt).
2. MISSING JSON SPEC (prompts/reconstruct-v0.2.md):
The v0.2 prompt template said 'matching the structure exactly' but never
defined what that structure was. The model invented its own field names
(input_classification, reasoning_mode, anchors) with snake_case instead of
camelCase, which failed Zod validation -> 500 errors.
Fix: Added explicit JSON schema section with exact key names, enum values,
and nested structure matching the Zod validation layer.
Additionally:
- Refactored route to use analyseScenario from lib/analysis (centralized)
- Added lib/analysis.js with shared analysis logic
- Updated components to display promptVersion and validation errors
- Added lib/reconstruction/prompt.js v0.1/v0.2 versioning
- Added lib/reconstruction/schema.js v0.2 Zod schemas
- Added debug tool scripts, evaluation results, and comparison findings
This commit is contained in:
@@ -8,7 +8,7 @@ const MAX_LENGTH = 10000;
|
||||
|
||||
export default function ScenarioForm() {
|
||||
const [scenario, setScenario] = useState("");
|
||||
const [status, setStatus] = useState("idle"); // idle | loading | error | success
|
||||
const [status, setStatus] = useState("idle"); // idle | loading | error | success | partial
|
||||
const [result, setResult] = useState(null);
|
||||
const textareaRef = useRef(null);
|
||||
|
||||
@@ -29,6 +29,10 @@ export default function ScenarioForm() {
|
||||
if (res.ok && data.validationStatus === "valid") {
|
||||
setStatus("success");
|
||||
setResult(data);
|
||||
} else if (data.success) {
|
||||
// Success in analysis but validation may be partial
|
||||
setStatus("success");
|
||||
setResult(data);
|
||||
} else {
|
||||
setStatus("error");
|
||||
setResult(data);
|
||||
@@ -39,8 +43,12 @@ export default function ScenarioForm() {
|
||||
}
|
||||
};
|
||||
|
||||
// Always show diagnostics when there's a result (even if validation failed)
|
||||
const hasDiagnostics = result && (result.reconstruction || result.modelName || result.responseDurationMs !== undefined);
|
||||
// Determine if we have meaningful content to display
|
||||
const hasClassification = result?.inputClassification;
|
||||
const hasReconstruction = result?.reconstruction;
|
||||
const hasNextQuestion = result?.nextQuestion;
|
||||
const hasEvidence = result?.evidence && result.evidence.length > 0;
|
||||
const hasMeaningfulContent = hasClassification || hasReconstruction || hasNextQuestion || hasEvidence;
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
@@ -65,6 +73,7 @@ export default function ScenarioForm() {
|
||||
</div>
|
||||
</form>
|
||||
|
||||
{/* Error state */}
|
||||
{status === "error" && (
|
||||
<div className="space-y-3">
|
||||
{result?.error && (
|
||||
@@ -72,36 +81,47 @@ export default function ScenarioForm() {
|
||||
Error: {result.error}
|
||||
</div>
|
||||
)}
|
||||
{hasDiagnostics && result?.modelName && (
|
||||
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1.5 text-sm">
|
||||
<dt className="text-gray-500">Model</dt>
|
||||
<dd>{result.modelName}</dd>
|
||||
<dt className="text-gray-500">Duration</dt>
|
||||
<dd>{result.responseDurationMs != null ? `${result.responseDurationMs}ms` : "?"}</dd>
|
||||
</dl>
|
||||
{/* Show partial content even on validation failure */}
|
||||
{(hasClassification || hasReconstruction) && (
|
||||
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
|
||||
⚠ Partial result — some fields failed validation. Showing what was accepted.
|
||||
</div>
|
||||
)}
|
||||
{hasReconstruction && (
|
||||
<ReconstructionView reconstruction={result} partial />
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{status === "success" && result?.reconstruction && (
|
||||
{/* Success state */}
|
||||
{status === "success" && hasMeaningfulContent && (
|
||||
<div className="space-y-4">
|
||||
<ReconstructionView reconstruction={result.reconstruction} />
|
||||
<DiagnosticsView result={result} />
|
||||
<ReconstructionView reconstruction={result} />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{status === "error" && result?.reconstruction && (
|
||||
<div className="space-y-3">
|
||||
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
|
||||
⚠ Partial result — some fields failed validation. Showing what was accepted.
|
||||
</div>
|
||||
<ReconstructionView reconstruction={result.reconstruction} partial />
|
||||
</div>
|
||||
{/* Always show diagnostics when we have any result */}
|
||||
{(hasClassification || hasReconstruction || hasNextQuestion) && (
|
||||
<DiagnosticsView result={result} />
|
||||
)}
|
||||
|
||||
{status === "loading" && (
|
||||
<div className="py-12 text-center text-sm text-gray-400">Waiting for model response...</div>
|
||||
)}
|
||||
|
||||
{/* Empty state */}
|
||||
{status === "idle" && (
|
||||
<div className="rounded-lg border border-dashed border-gray-300 bg-gray-50 px-6 py-8 text-center">
|
||||
<p className="text-sm text-gray-400">Enter a scenario above and click Analyse to begin.</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Invalid result with no partial data */}
|
||||
{status === "error" && !result?.error && !hasMeaningfulContent && (
|
||||
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
|
||||
Validation failed — no structured output was produced.
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user