robbond
956fc2e31e
Two bugs were causing the model to return {"status":"ok"} / {"status":"ready"}
instead of structured reconstruction data, resulting in POST /api/analyse 500:
1. DOUBLE-WRAPPING BUG (lib/llm/provider.js):
generateReconstruction() called buildPrompt(scenario) on input that was
already a fully-built prompt string from analyseScenario(). This wrapped the
v0.1 prompt (~5000+ chars) in another template layer, producing incomprehensible
output that the model could not parse as structured JSON.
Fix: Pass scenario through directly (it is ALREADY a built prompt).
2. MISSING JSON SPEC (prompts/reconstruct-v0.2.md):
The v0.2 prompt template said 'matching the structure exactly' but never
defined what that structure was. The model invented its own field names
(input_classification, reasoning_mode, anchors) with snake_case instead of
camelCase, which failed Zod validation -> 500 errors.
Fix: Added explicit JSON schema section with exact key names, enum values,
and nested structure matching the Zod validation layer.
Additionally:
- Refactored route to use analyseScenario from lib/analysis (centralized)
- Added lib/analysis.js with shared analysis logic
- Updated components to display promptVersion and validation errors
- Added lib/reconstruction/prompt.js v0.1/v0.2 versioning
- Added lib/reconstruction/schema.js v0.2 Zod schemas
- Added debug tool scripts, evaluation results, and comparison findings
331 lines
9.8 KiB
JSON
331 lines
9.8 KiB
JSON
{
|
|
"timestamp": "2026-08-01T06:11:41.538Z",
|
|
"provider": "ollama-real",
|
|
"promptVersion": "v0.2",
|
|
"casesRun": 10,
|
|
"summary": {
|
|
"technical": {
|
|
"schemaValidityRate": "0.0%",
|
|
"classificationMatchRate": "0.0%",
|
|
"nextQuestionPresentRate": "0.0%",
|
|
"passRate": "0.0%"
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConceptMatchRate": "100.0%",
|
|
"unsupportedInferenceFailures": "0",
|
|
"passRate": "0.0%"
|
|
},
|
|
"combinedPassRate": "0.0%",
|
|
"averageResponseDurationMs": "19081"
|
|
},
|
|
"testCaseResults": [
|
|
{
|
|
"id": "diag-01",
|
|
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
|
"responseDurationMs": 19459,
|
|
"actualPrimaryType": null,
|
|
"actualReasoningModes": [],
|
|
"technical": {
|
|
"schemaValid": false,
|
|
"classificationMatch": false,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": false,
|
|
"pass": false,
|
|
"errors": [
|
|
"inputClassification: Required",
|
|
"reconstruction: Required",
|
|
"evidence: Required",
|
|
"nextQuestion: Required"
|
|
]
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "diag-02",
|
|
"input": "Some customers reported that the new app crashes when uploading photos.",
|
|
"responseDurationMs": 14693,
|
|
"actualPrimaryType": null,
|
|
"actualReasoningModes": [],
|
|
"technical": {
|
|
"schemaValid": false,
|
|
"classificationMatch": false,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": false,
|
|
"pass": false,
|
|
"errors": [
|
|
"Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download",
|
|
"Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download"
|
|
]
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "diag-03",
|
|
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
|
"responseDurationMs": 13516,
|
|
"actualPrimaryType": null,
|
|
"actualReasoningModes": [],
|
|
"technical": {
|
|
"schemaValid": false,
|
|
"classificationMatch": false,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": false,
|
|
"pass": false,
|
|
"errors": [
|
|
"inputClassification: Required",
|
|
"reconstruction: Required",
|
|
"evidence: Required",
|
|
"nextQuestion: Required"
|
|
]
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "diag-04",
|
|
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
|
"responseDurationMs": 48851,
|
|
"actualPrimaryType": null,
|
|
"actualReasoningModes": [],
|
|
"technical": {
|
|
"schemaValid": false,
|
|
"classificationMatch": false,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": false,
|
|
"pass": false,
|
|
"errors": [
|
|
"inputClassification: Required",
|
|
"reconstruction: Required",
|
|
"evidence: Required",
|
|
"nextQuestion: Required"
|
|
]
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "diag-05",
|
|
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
|
"responseDurationMs": 16508,
|
|
"actualPrimaryType": null,
|
|
"actualReasoningModes": [],
|
|
"technical": {
|
|
"schemaValid": false,
|
|
"classificationMatch": false,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": false,
|
|
"pass": false,
|
|
"errors": [
|
|
"inputClassification: Required",
|
|
"reconstruction: Required",
|
|
"evidence: Required",
|
|
"nextQuestion: Required"
|
|
]
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "diag-06",
|
|
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
|
"responseDurationMs": 15608,
|
|
"actualPrimaryType": null,
|
|
"actualReasoningModes": [],
|
|
"technical": {
|
|
"schemaValid": false,
|
|
"classificationMatch": false,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": false,
|
|
"pass": false,
|
|
"errors": [
|
|
"inputClassification: Required",
|
|
"reconstruction: Required",
|
|
"evidence: Required",
|
|
"nextQuestion: Required"
|
|
]
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "diag-07",
|
|
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
|
"responseDurationMs": 15750,
|
|
"actualPrimaryType": null,
|
|
"actualReasoningModes": [],
|
|
"technical": {
|
|
"schemaValid": false,
|
|
"classificationMatch": false,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": false,
|
|
"pass": false,
|
|
"errors": [
|
|
"inputClassification: Required",
|
|
"reconstruction: Required",
|
|
"evidence: Required",
|
|
"nextQuestion: Required"
|
|
]
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "diag-08",
|
|
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
|
"responseDurationMs": 15383,
|
|
"actualPrimaryType": null,
|
|
"actualReasoningModes": [],
|
|
"technical": {
|
|
"schemaValid": false,
|
|
"classificationMatch": false,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": false,
|
|
"pass": false,
|
|
"errors": [
|
|
"inputClassification: Required",
|
|
"reconstruction: Required",
|
|
"evidence: Required",
|
|
"nextQuestion: Required"
|
|
]
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "diag-09",
|
|
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
|
"responseDurationMs": 15643,
|
|
"actualPrimaryType": null,
|
|
"actualReasoningModes": [],
|
|
"technical": {
|
|
"schemaValid": false,
|
|
"classificationMatch": false,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": false,
|
|
"pass": false,
|
|
"errors": [
|
|
"inputClassification: Required",
|
|
"reconstruction: Required",
|
|
"evidence: Required",
|
|
"nextQuestion: Required"
|
|
]
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "diag-10",
|
|
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
|
"responseDurationMs": 15401,
|
|
"actualPrimaryType": null,
|
|
"actualReasoningModes": [],
|
|
"technical": {
|
|
"schemaValid": false,
|
|
"classificationMatch": false,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": false,
|
|
"pass": false,
|
|
"errors": [
|
|
"inputClassification: Required",
|
|
"reconstruction: Required",
|
|
"evidence: Required",
|
|
"nextQuestion: Required"
|
|
]
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"pass": false
|
|
}
|
|
}
|
|
]
|
|
} |