robbond
956fc2e31e
Two bugs were causing the model to return {"status":"ok"} / {"status":"ready"}
instead of structured reconstruction data, resulting in POST /api/analyse 500:
1. DOUBLE-WRAPPING BUG (lib/llm/provider.js):
generateReconstruction() called buildPrompt(scenario) on input that was
already a fully-built prompt string from analyseScenario(). This wrapped the
v0.1 prompt (~5000+ chars) in another template layer, producing incomprehensible
output that the model could not parse as structured JSON.
Fix: Pass scenario through directly (it is ALREADY a built prompt).
2. MISSING JSON SPEC (prompts/reconstruct-v0.2.md):
The v0.2 prompt template said 'matching the structure exactly' but never
defined what that structure was. The model invented its own field names
(input_classification, reasoning_mode, anchors) with snake_case instead of
camelCase, which failed Zod validation -> 500 errors.
Fix: Added explicit JSON schema section with exact key names, enum values,
and nested structure matching the Zod validation layer.
Additionally:
- Refactored route to use analyseScenario from lib/analysis (centralized)
- Added lib/analysis.js with shared analysis logic
- Updated components to display promptVersion and validation errors
- Added lib/reconstruction/prompt.js v0.1/v0.2 versioning
- Added lib/reconstruction/schema.js v0.2 Zod schemas
- Added debug tool scripts, evaluation results, and comparison findings
1735 lines
43 KiB
JSON
1735 lines
43 KiB
JSON
{
|
|
"timestamp": "2026-08-01T05:31:49.869Z",
|
|
"provider": "mock",
|
|
"promptVersion": "v0.2",
|
|
"casesRun": 35,
|
|
"summary": {
|
|
"technical": {
|
|
"schemaValidityRate": "100.0%",
|
|
"classificationMatchRate": "100.0%",
|
|
"nextQuestionPresentRate": "100.0%",
|
|
"passRate": "100.0%"
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConceptMatchRate": "0.0%",
|
|
"unsupportedInferenceFailures": "0",
|
|
"passRate": "0.0%"
|
|
},
|
|
"combinedPassRate": "0.0%",
|
|
"averageResponseDurationMs": "1"
|
|
},
|
|
"testCaseResults": [
|
|
{
|
|
"id": "tc-001",
|
|
"input": "All customers cannot download their invoices.",
|
|
"responseDurationMs": 5,
|
|
"actualPrimaryType": "observed_problem",
|
|
"actualReasoningModes": [
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "customers",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "invoices",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "download",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "access_issue",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-002",
|
|
"input": "Some customers cannot download their invoices.",
|
|
"responseDurationMs": 2,
|
|
"actualPrimaryType": "observed_problem",
|
|
"actualReasoningModes": [
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "some_customers",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "invoices",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "download",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "root_cause",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "payment_system_failure",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-003",
|
|
"input": "Complaints increased by 35%.",
|
|
"responseDurationMs": 1,
|
|
"actualPrimaryType": "unexplained_change",
|
|
"actualReasoningModes": [
|
|
"establish_baseline",
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "complaints",
|
|
"found": true
|
|
},
|
|
{
|
|
"concept": "increase",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "35_percent",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "cause_of_complaints",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "customer_dissatisfaction_is_worse",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-004",
|
|
"input": "Complaints increased by 35% while production increased by 40%.",
|
|
"responseDurationMs": 1,
|
|
"actualPrimaryType": "unexplained_change",
|
|
"actualReasoningModes": [
|
|
"establish_baseline",
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "complaints_increase",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "production_increase",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "relative_rates",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "production_quality_declined",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-005",
|
|
"input": "Sales are falling.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "observed_problem",
|
|
"actualReasoningModes": [
|
|
"establish_baseline",
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "sales_decline",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "direction_negative",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "cause_of_fall",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "competitor_action",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-006",
|
|
"input": "Sales fell sharply immediately after the price increase.",
|
|
"responseDurationMs": 1,
|
|
"actualPrimaryType": "causal_claim",
|
|
"actualReasoningModes": [
|
|
"establish_baseline",
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "sales_decline",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "price_increase",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "temporal_correlation",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "price_increase_caused_the_fall",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-007",
|
|
"input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "contradiction",
|
|
"actualReasoningModes": [
|
|
"investigate_contradiction",
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "revenue_above_target",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "profit_decline",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "divergence",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "cost_overrun_is_the_cause",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-008",
|
|
"input": "Revenue from the premium tier dropped while total revenue grew.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "unexplained_change",
|
|
"actualReasoningModes": [
|
|
"identify_difference",
|
|
"establish_baseline",
|
|
"validate_measurement"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "premium_tier_decline",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "total_revenue_growth",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "segment_cannibalization_risk",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "pricing_change_occurred",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-009",
|
|
"input": "We need to improve our customer retention rate.",
|
|
"responseDurationMs": 1,
|
|
"actualPrimaryType": "decision_request",
|
|
"actualReasoningModes": [
|
|
"identify_difference",
|
|
"decision_support",
|
|
"identify_missing_information"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "retention_improvement_desired",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "current_state_unknown",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "retention_rate_is_low",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "churn_has_increased",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-010",
|
|
"input": "The system latency went from 200ms to 5 seconds on Tuesday.",
|
|
"responseDurationMs": 1,
|
|
"actualPrimaryType": "unexplained_change",
|
|
"actualReasoningModes": [
|
|
"identify_difference",
|
|
"establish_baseline",
|
|
"validate_measurement"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "latency_baseline_200ms",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "latency_spike_5s",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "timestamp_tuesday",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "database_cause",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "release_cause",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-011",
|
|
"input": "The new release should fix the login issue.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "decision_request",
|
|
"actualReasoningModes": [
|
|
"identify_difference",
|
|
"decision_support",
|
|
"identify_missing_information"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "proposed_solution",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "login_issue",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "solution_claim",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "login_issue_is_real",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "release_will_work",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-012",
|
|
"input": "I think therefore I am.",
|
|
"responseDurationMs": 1,
|
|
"actualPrimaryType": "ambiguous_statement",
|
|
"actualReasoningModes": [
|
|
"identify_difference",
|
|
"clarify_meaning"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "philosophical_statement",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "insufficient_operational_context",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "business_problem_exists",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "actionable_insight_possible",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-013",
|
|
"input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "ambiguous_statement",
|
|
"actualReasoningModes": [
|
|
"identify_difference",
|
|
"clarify_meaning"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "meta_context",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "testing_hypothesis",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "self_reference",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": []
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-014",
|
|
"input": "The warehouse manager reported that inventory counts don't match the system.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "reported_claim",
|
|
"actualReasoningModes": [
|
|
"identify_difference",
|
|
"validate_claim"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "warehouse_manager_report",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "inventory_mismatch",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "system_discrepancy",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "source_attribution",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "theft_occurred",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "software_bug",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-015",
|
|
"input": "We've seen a 35% increase in customer complaints.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "unexplained_change",
|
|
"actualReasoningModes": [
|
|
"establish_baseline",
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "complaints_increase",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "percentage_metric",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "product_quality_declined",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "customer_satisfaction_drop",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-016",
|
|
"input": "The number of active users increased by 500%, from 4 to 2,001.",
|
|
"responseDurationMs": 1,
|
|
"actualPrimaryType": "unexplained_change",
|
|
"actualReasoningModes": [
|
|
"identify_difference",
|
|
"establish_baseline",
|
|
"validate_measurement"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "active_users_metric",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "absolute_vs_relative_growth",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "small_base_problem",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "product_success",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-017",
|
|
"input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "contradiction",
|
|
"actualReasoningModes": [
|
|
"investigate_contradiction",
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "engagement_improvement",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "support_backlog_growth",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "divergent_metrics",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "users_are_angry",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "product_quality_is_worse",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-018",
|
|
"input": "The manufacturing team needs better quality control.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "decision_request",
|
|
"actualReasoningModes": [
|
|
"identify_difference",
|
|
"decision_support",
|
|
"identify_missing_information"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "manufacturing_team",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "quality_control_desired",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "quality_is_bad",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "defect_rate_is_high",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-019",
|
|
"input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
|
|
"responseDurationMs": 1,
|
|
"actualPrimaryType": "observed_problem",
|
|
"actualReasoningModes": [
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "eu_region",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "403_error",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "access_denied",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "geographic_scope",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "gdpr_cause",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "regulatory_change",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-020",
|
|
"input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "observed_problem",
|
|
"actualReasoningModes": [
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "eu_region_subset",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "403_error",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "partial_reachability",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "all_eu_users_affected",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-021",
|
|
"input": "Production output was 1,200 units last month and 1,180 units this month.",
|
|
"responseDurationMs": 1,
|
|
"actualPrimaryType": "unexplained_change",
|
|
"actualReasoningModes": [
|
|
"identify_difference",
|
|
"establish_baseline",
|
|
"validate_measurement"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "production_output",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "month_over_month_decline",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "absolute_difference",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "efficiency_loss_occurred",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "equipment_failure",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-022",
|
|
"input": "The CFO reported that the company's cash position is healthy.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "reported_claim",
|
|
"actualReasoningModes": [
|
|
"identify_difference",
|
|
"validate_claim"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "cfo_statement",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "cash_position_claim",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "source_attribution_cfo",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "cash_is_healthy",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "financial_stability_is_real",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-023",
|
|
"input": "We have enough funding to operate for 18 months.",
|
|
"responseDurationMs": 1,
|
|
"actualPrimaryType": "observed_problem",
|
|
"actualReasoningModes": [
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "funding_period",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "operational_sustainability",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "burn_rate_unknown",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "no_risk_exists",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-024",
|
|
"input": "The new feature was deployed at 3am and user complaints tripled the next day.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "causal_claim",
|
|
"actualReasoningModes": [
|
|
"establish_baseline",
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "feature_deployment",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "timing_3am",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "complaint_tripling",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "temporal_relationship",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "deployment_caused_complaints",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-025",
|
|
"input": "We need to launch a mobile app to capture market share.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "decision_request",
|
|
"actualReasoningModes": [
|
|
"identify_difference",
|
|
"decision_support",
|
|
"identify_missing_information"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "mobile_app_proposed",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "market_share_desired",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "no_mobile_app_exists",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "competitors_have_apps",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-026",
|
|
"input": "The system has been running for 90 days without failure since the migration.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "observed_problem",
|
|
"actualReasoningModes": [
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "uptime_90_days",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "post_migration_context",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "baseline_established",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "system_is_stable_forever",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-027",
|
|
"input": "No one has submitted the required compliance report despite multiple reminders.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "observed_problem",
|
|
"actualReasoningModes": [
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "compliance_report",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "multiple_reminders",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "non_submission",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "absent_action",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "deliberate_refusal",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "negligence",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-028",
|
|
"input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "observed_problem",
|
|
"actualReasoningModes": [
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "audit_findings",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "incorrect_reports_rate_3_of_10",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "data_accuracy_issue",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "intentional_falsification",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "systemic_failure",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-029",
|
|
"input": "We should implement the new CRM because our competitors have one.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "decision_request",
|
|
"actualReasoningModes": [
|
|
"identify_difference",
|
|
"decision_support",
|
|
"identify_missing_information"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "crm_proposal",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "competitor_comparison",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "competitive_pressure",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "crm_will_help",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "we_lack_crm",
|
|
"absent": true
|
|
},
|
|
{
|
|
"concept": "competitors_success_is_from_crm",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-030",
|
|
"input": "The server response time was acceptable last quarter but degraded this month.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "observed_problem",
|
|
"actualReasoningModes": [
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "response_time_baseline_acceptable",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "degradation_timeline",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "quarter_to_month_comparison",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "load_increase_occurred",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-031",
|
|
"input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "observed_problem",
|
|
"actualReasoningModes": [
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "regulatory_requirement",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "data_location_violation",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "cross_border_backup",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "compliance_failure_is_certain",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-032",
|
|
"input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
|
|
"responseDurationMs": 1,
|
|
"actualPrimaryType": "causal_claim",
|
|
"actualReasoningModes": [
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "industry_decline_prediction",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "external_source",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "regulatory_cause",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "15_percent_forecast",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "decline_will_occur",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-033",
|
|
"input": "The database schema was changed on Friday but the reports are still working.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "unexplained_change",
|
|
"actualReasoningModes": [
|
|
"identify_difference",
|
|
"establish_baseline",
|
|
"validate_measurement"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "schema_change",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "reports_working_post_change",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "unexpected_continuity",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "change_was_harmless",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-034",
|
|
"input": "Some team members say the new process is better while others say it's slower.",
|
|
"responseDurationMs": 0,
|
|
"actualPrimaryType": "observed_problem",
|
|
"actualReasoningModes": [
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": false,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "subjective_split",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "new_process_evaluation",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "conflicting_opinions",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "measurement_gap",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "process_is_better_or_worse",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"id": "tc-035",
|
|
"input": "The application works fine on Chrome but not on Safari.",
|
|
"responseDurationMs": 1,
|
|
"actualPrimaryType": "observed_problem",
|
|
"actualReasoningModes": [
|
|
"identify_difference"
|
|
],
|
|
"technical": {
|
|
"schemaValid": true,
|
|
"classificationMatch": true,
|
|
"reasoningModeMatch": true,
|
|
"nextQuestionPresent": true,
|
|
"pass": true,
|
|
"errors": []
|
|
},
|
|
"reasoningQuality": {
|
|
"requiredConcepts": {
|
|
"pass": false,
|
|
"details": [
|
|
{
|
|
"concept": "chrome_compatibility",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "safari_incompatibility",
|
|
"found": false
|
|
},
|
|
{
|
|
"concept": "browser_specific_issue",
|
|
"found": false
|
|
}
|
|
]
|
|
},
|
|
"unsupportedInferencesAbsent": {
|
|
"pass": true,
|
|
"details": [
|
|
{
|
|
"concept": "webkit_bug",
|
|
"absent": true
|
|
}
|
|
]
|
|
},
|
|
"pass": false
|
|
}
|
|
}
|
|
]
|
|
} |