{ "timestamp": "2026-08-01T06:02:26.238Z", "provider": "mock", "promptVersion": "v0.2", "casesRun": 35, "summary": { "technical": { "schemaValidityRate": "100.0%", "classificationMatchRate": "100.0%", "nextQuestionPresentRate": "100.0%", "passRate": "100.0%" }, "reasoningQuality": { "requiredConceptMatchRate": "0.0%", "unsupportedInferenceFailures": "0", "passRate": "0.0%" }, "combinedPassRate": "0.0%", "averageResponseDurationMs": "1" }, "testCaseResults": [ { "id": "tc-001", "input": "All customers cannot download their invoices.", "responseDurationMs": 4, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "customers", "found": false }, { "concept": "invoices", "found": false }, { "concept": "download", "found": false }, { "concept": "access_issue", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [] }, "pass": false } }, { "id": "tc-002", "input": "Some customers cannot download their invoices.", "responseDurationMs": 1, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "some_customers", "found": false }, { "concept": "invoices", "found": false }, { "concept": "download", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "root_cause", "absent": true }, { "concept": "payment_system_failure", "absent": true } ] }, "pass": false } }, { "id": "tc-003", "input": "Complaints increased by 35%.", "responseDurationMs": 1, "actualPrimaryType": "unexplained_change", "actualReasoningModes": [ "establish_baseline", "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "complaints", "found": true }, { "concept": "increase", "found": false }, { "concept": "35_percent", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "cause_of_complaints", "absent": true }, { "concept": "customer_dissatisfaction_is_worse", "absent": true } ] }, "pass": false } }, { "id": "tc-004", "input": "Complaints increased by 35% while production increased by 40%.", "responseDurationMs": 1, "actualPrimaryType": "unexplained_change", "actualReasoningModes": [ "establish_baseline", "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "complaints_increase", "found": false }, { "concept": "production_increase", "found": false }, { "concept": "relative_rates", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "production_quality_declined", "absent": true } ] }, "pass": false } }, { "id": "tc-005", "input": "Sales are falling.", "responseDurationMs": 1, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "establish_baseline", "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "sales_decline", "found": false }, { "concept": "direction_negative", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "cause_of_fall", "absent": true }, { "concept": "competitor_action", "absent": true } ] }, "pass": false } }, { "id": "tc-006", "input": "Sales fell sharply immediately after the price increase.", "responseDurationMs": 1, "actualPrimaryType": "causal_claim", "actualReasoningModes": [ "establish_baseline", "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "sales_decline", "found": false }, { "concept": "price_increase", "found": false }, { "concept": "temporal_correlation", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "price_increase_caused_the_fall", "absent": true } ] }, "pass": false } }, { "id": "tc-007", "input": "The quarterly revenue exceeded targets but net profit declined by 12%.", "responseDurationMs": 1, "actualPrimaryType": "contradiction", "actualReasoningModes": [ "investigate_contradiction", "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "revenue_above_target", "found": false }, { "concept": "profit_decline", "found": false }, { "concept": "divergence", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "cost_overrun_is_the_cause", "absent": true } ] }, "pass": false } }, { "id": "tc-008", "input": "Revenue from the premium tier dropped while total revenue grew.", "responseDurationMs": 0, "actualPrimaryType": "unexplained_change", "actualReasoningModes": [ "identify_difference", "establish_baseline", "validate_measurement" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "premium_tier_decline", "found": false }, { "concept": "total_revenue_growth", "found": false }, { "concept": "segment_cannibalization_risk", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "pricing_change_occurred", "absent": true } ] }, "pass": false } }, { "id": "tc-009", "input": "We need to improve our customer retention rate.", "responseDurationMs": 0, "actualPrimaryType": "decision_request", "actualReasoningModes": [ "identify_difference", "decision_support", "identify_missing_information" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "retention_improvement_desired", "found": false }, { "concept": "current_state_unknown", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "retention_rate_is_low", "absent": true }, { "concept": "churn_has_increased", "absent": true } ] }, "pass": false } }, { "id": "tc-010", "input": "The system latency went from 200ms to 5 seconds on Tuesday.", "responseDurationMs": 1, "actualPrimaryType": "unexplained_change", "actualReasoningModes": [ "identify_difference", "establish_baseline", "validate_measurement" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "latency_baseline_200ms", "found": false }, { "concept": "latency_spike_5s", "found": false }, { "concept": "timestamp_tuesday", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "database_cause", "absent": true }, { "concept": "release_cause", "absent": true } ] }, "pass": false } }, { "id": "tc-011", "input": "The new release should fix the login issue.", "responseDurationMs": 0, "actualPrimaryType": "decision_request", "actualReasoningModes": [ "identify_difference", "decision_support", "identify_missing_information" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "proposed_solution", "found": false }, { "concept": "login_issue", "found": false }, { "concept": "solution_claim", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "login_issue_is_real", "absent": true }, { "concept": "release_will_work", "absent": true } ] }, "pass": false } }, { "id": "tc-012", "input": "I think therefore I am.", "responseDurationMs": 1, "actualPrimaryType": "ambiguous_statement", "actualReasoningModes": [ "identify_difference", "clarify_meaning" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "philosophical_statement", "found": false }, { "concept": "insufficient_operational_context", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "business_problem_exists", "absent": true }, { "concept": "actionable_insight_possible", "absent": true } ] }, "pass": false } }, { "id": "tc-013", "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.", "responseDurationMs": 0, "actualPrimaryType": "ambiguous_statement", "actualReasoningModes": [ "identify_difference", "clarify_meaning" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "meta_context", "found": false }, { "concept": "testing_hypothesis", "found": false }, { "concept": "self_reference", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [] }, "pass": false } }, { "id": "tc-014", "input": "The warehouse manager reported that inventory counts don't match the system.", "responseDurationMs": 0, "actualPrimaryType": "reported_claim", "actualReasoningModes": [ "identify_difference", "validate_claim" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "warehouse_manager_report", "found": false }, { "concept": "inventory_mismatch", "found": false }, { "concept": "system_discrepancy", "found": false }, { "concept": "source_attribution", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "theft_occurred", "absent": true }, { "concept": "software_bug", "absent": true } ] }, "pass": false } }, { "id": "tc-015", "input": "We've seen a 35% increase in customer complaints.", "responseDurationMs": 1, "actualPrimaryType": "unexplained_change", "actualReasoningModes": [ "establish_baseline", "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "complaints_increase", "found": false }, { "concept": "percentage_metric", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "product_quality_declined", "absent": true }, { "concept": "customer_satisfaction_drop", "absent": true } ] }, "pass": false } }, { "id": "tc-016", "input": "The number of active users increased by 500%, from 4 to 2,001.", "responseDurationMs": 0, "actualPrimaryType": "unexplained_change", "actualReasoningModes": [ "identify_difference", "establish_baseline", "validate_measurement" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "active_users_metric", "found": false }, { "concept": "absolute_vs_relative_growth", "found": false }, { "concept": "small_base_problem", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "product_success", "absent": true } ] }, "pass": false } }, { "id": "tc-017", "input": "User engagement metrics improved but the support ticket backlog grew by 200%.", "responseDurationMs": 0, "actualPrimaryType": "contradiction", "actualReasoningModes": [ "investigate_contradiction", "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "engagement_improvement", "found": false }, { "concept": "support_backlog_growth", "found": false }, { "concept": "divergent_metrics", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "users_are_angry", "absent": true }, { "concept": "product_quality_is_worse", "absent": true } ] }, "pass": false } }, { "id": "tc-018", "input": "The manufacturing team needs better quality control.", "responseDurationMs": 1, "actualPrimaryType": "decision_request", "actualReasoningModes": [ "identify_difference", "decision_support", "identify_missing_information" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "manufacturing_team", "found": false }, { "concept": "quality_control_desired", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "quality_is_bad", "absent": true }, { "concept": "defect_rate_is_high", "absent": true } ] }, "pass": false } }, { "id": "tc-019", "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.", "responseDurationMs": 0, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "eu_region", "found": false }, { "concept": "403_error", "found": false }, { "concept": "access_denied", "found": false }, { "concept": "geographic_scope", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "gdpr_cause", "absent": true }, { "concept": "regulatory_change", "absent": true } ] }, "pass": false } }, { "id": "tc-020", "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.", "responseDurationMs": 0, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "eu_region_subset", "found": false }, { "concept": "403_error", "found": false }, { "concept": "partial_reachability", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "all_eu_users_affected", "absent": true } ] }, "pass": false } }, { "id": "tc-021", "input": "Production output was 1,200 units last month and 1,180 units this month.", "responseDurationMs": 0, "actualPrimaryType": "unexplained_change", "actualReasoningModes": [ "identify_difference", "establish_baseline", "validate_measurement" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "production_output", "found": false }, { "concept": "month_over_month_decline", "found": false }, { "concept": "absolute_difference", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "efficiency_loss_occurred", "absent": true }, { "concept": "equipment_failure", "absent": true } ] }, "pass": false } }, { "id": "tc-022", "input": "The CFO reported that the company's cash position is healthy.", "responseDurationMs": 0, "actualPrimaryType": "reported_claim", "actualReasoningModes": [ "identify_difference", "validate_claim" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "cfo_statement", "found": false }, { "concept": "cash_position_claim", "found": false }, { "concept": "source_attribution_cfo", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "cash_is_healthy", "absent": true }, { "concept": "financial_stability_is_real", "absent": true } ] }, "pass": false } }, { "id": "tc-023", "input": "We have enough funding to operate for 18 months.", "responseDurationMs": 1, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "funding_period", "found": false }, { "concept": "operational_sustainability", "found": false }, { "concept": "burn_rate_unknown", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "no_risk_exists", "absent": true } ] }, "pass": false } }, { "id": "tc-024", "input": "The new feature was deployed at 3am and user complaints tripled the next day.", "responseDurationMs": 0, "actualPrimaryType": "causal_claim", "actualReasoningModes": [ "establish_baseline", "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "feature_deployment", "found": false }, { "concept": "timing_3am", "found": false }, { "concept": "complaint_tripling", "found": false }, { "concept": "temporal_relationship", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "deployment_caused_complaints", "absent": true } ] }, "pass": false } }, { "id": "tc-025", "input": "We need to launch a mobile app to capture market share.", "responseDurationMs": 0, "actualPrimaryType": "decision_request", "actualReasoningModes": [ "identify_difference", "decision_support", "identify_missing_information" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "mobile_app_proposed", "found": false }, { "concept": "market_share_desired", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "no_mobile_app_exists", "absent": true }, { "concept": "competitors_have_apps", "absent": true } ] }, "pass": false } }, { "id": "tc-026", "input": "The system has been running for 90 days without failure since the migration.", "responseDurationMs": 1, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "uptime_90_days", "found": false }, { "concept": "post_migration_context", "found": false }, { "concept": "baseline_established", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "system_is_stable_forever", "absent": true } ] }, "pass": false } }, { "id": "tc-027", "input": "No one has submitted the required compliance report despite multiple reminders.", "responseDurationMs": 0, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "compliance_report", "found": false }, { "concept": "multiple_reminders", "found": false }, { "concept": "non_submission", "found": false }, { "concept": "absent_action", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "deliberate_refusal", "absent": true }, { "concept": "negligence", "absent": true } ] }, "pass": false } }, { "id": "tc-028", "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.", "responseDurationMs": 0, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "audit_findings", "found": false }, { "concept": "incorrect_reports_rate_3_of_10", "found": false }, { "concept": "data_accuracy_issue", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "intentional_falsification", "absent": true }, { "concept": "systemic_failure", "absent": true } ] }, "pass": false } }, { "id": "tc-029", "input": "We should implement the new CRM because our competitors have one.", "responseDurationMs": 0, "actualPrimaryType": "decision_request", "actualReasoningModes": [ "identify_difference", "decision_support", "identify_missing_information" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "crm_proposal", "found": false }, { "concept": "competitor_comparison", "found": false }, { "concept": "competitive_pressure", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "crm_will_help", "absent": true }, { "concept": "we_lack_crm", "absent": true }, { "concept": "competitors_success_is_from_crm", "absent": true } ] }, "pass": false } }, { "id": "tc-030", "input": "The server response time was acceptable last quarter but degraded this month.", "responseDurationMs": 1, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "response_time_baseline_acceptable", "found": false }, { "concept": "degradation_timeline", "found": false }, { "concept": "quarter_to_month_comparison", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "load_increase_occurred", "absent": true } ] }, "pass": false } }, { "id": "tc-031", "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.", "responseDurationMs": 0, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "regulatory_requirement", "found": false }, { "concept": "data_location_violation", "found": false }, { "concept": "cross_border_backup", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "compliance_failure_is_certain", "absent": true } ] }, "pass": false } }, { "id": "tc-032", "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.", "responseDurationMs": 0, "actualPrimaryType": "causal_claim", "actualReasoningModes": [ "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "industry_decline_prediction", "found": false }, { "concept": "external_source", "found": false }, { "concept": "regulatory_cause", "found": false }, { "concept": "15_percent_forecast", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "decline_will_occur", "absent": true } ] }, "pass": false } }, { "id": "tc-033", "input": "The database schema was changed on Friday but the reports are still working.", "responseDurationMs": 0, "actualPrimaryType": "unexplained_change", "actualReasoningModes": [ "identify_difference", "establish_baseline", "validate_measurement" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "schema_change", "found": false }, { "concept": "reports_working_post_change", "found": false }, { "concept": "unexpected_continuity", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "change_was_harmless", "absent": true } ] }, "pass": false } }, { "id": "tc-034", "input": "Some team members say the new process is better while others say it's slower.", "responseDurationMs": 1, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "subjective_split", "found": false }, { "concept": "new_process_evaluation", "found": false }, { "concept": "conflicting_opinions", "found": false }, { "concept": "measurement_gap", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "process_is_better_or_worse", "absent": true } ] }, "pass": false } }, { "id": "tc-035", "input": "The application works fine on Chrome but not on Safari.", "responseDurationMs": 0, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "technical": { "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "nextQuestionPresent": true, "pass": true, "errors": [] }, "reasoningQuality": { "requiredConcepts": { "pass": false, "details": [ { "concept": "chrome_compatibility", "found": false }, { "concept": "safari_incompatibility", "found": false }, { "concept": "browser_specific_issue", "found": false } ] }, "unsupportedInferencesAbsent": { "pass": true, "details": [ { "concept": "webkit_bug", "absent": true } ] }, "pass": false } } ] }