{ "timestamp": "2026-07-31T18:34:17.855Z", "provider": "mock", "promptVersion": "v0.2", "casesRun": 35, "summary": { "schemaValidityRate": "100.0%", "classificationMatchRate": "82.9%", "reasoningModeMatchRate": "51.4%", "requiredConceptMatchRate": "0.0%", "unsupportedInferenceFailures": "0", "averageResponseDurationMs": "1", "fullPassRate": "82.9%" }, "testCaseResults": [ { "id": "tc-001", "input": "All customers cannot download their invoices.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 5, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "errors": [] }, { "id": "tc-002", "input": "Some customers cannot download their invoices.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 2, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "errors": [] }, { "id": "tc-003", "input": "Complaints increased by 35%.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 1, "actualPrimaryType": "unexplained_change", "actualReasoningModes": [ "establish_baseline", "identify_difference" ], "errors": [] }, { "id": "tc-004", "input": "Complaints increased by 35% while production increased by 40%.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "unexplained_change", "actualReasoningModes": [ "establish_baseline", "identify_difference" ], "errors": [] }, { "id": "tc-005", "input": "Sales are falling.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 1, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "establish_baseline", "identify_difference" ], "errors": [] }, { "id": "tc-006", "input": "Sales fell sharply immediately after the price increase.", "pass": false, "schemaValid": true, "classificationMatch": false, "reasoningModeMatch": false, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 1, "actualPrimaryType": "unexplained_change", "actualReasoningModes": [ "establish_baseline", "identify_difference" ], "errors": [] }, { "id": "tc-007", "input": "The quarterly revenue exceeded targets but net profit declined by 12%.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 1, "actualPrimaryType": "contradiction", "actualReasoningModes": [ "investigate_contradiction", "identify_difference" ], "errors": [] }, { "id": "tc-008", "input": "Revenue from the premium tier dropped while total revenue grew.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 1, "actualPrimaryType": "unexplained_change", "actualReasoningModes": [ "identify_difference", "establish_baseline", "validate_measurement" ], "errors": [] }, { "id": "tc-009", "input": "We need to improve our customer retention rate.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "decision_request", "actualReasoningModes": [ "identify_difference", "decision_support", "identify_missing_information" ], "errors": [] }, { "id": "tc-010", "input": "The system latency went from 200ms to 5 seconds on Tuesday.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "unexplained_change", "actualReasoningModes": [ "identify_difference", "establish_baseline", "validate_measurement" ], "errors": [] }, { "id": "tc-011", "input": "The new release should fix the login issue.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 1, "actualPrimaryType": "decision_request", "actualReasoningModes": [ "identify_difference", "decision_support", "identify_missing_information" ], "errors": [] }, { "id": "tc-012", "input": "I think therefore I am.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "ambiguous_statement", "actualReasoningModes": [ "identify_difference", "clarify_meaning" ], "errors": [] }, { "id": "tc-013", "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 1, "actualPrimaryType": "ambiguous_statement", "actualReasoningModes": [ "identify_difference", "clarify_meaning" ], "errors": [] }, { "id": "tc-014", "input": "The warehouse manager reported that inventory counts don't match the system.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "reported_claim", "actualReasoningModes": [ "identify_difference", "validate_claim" ], "errors": [] }, { "id": "tc-015", "input": "We've seen a 35% increase in customer complaints.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 1, "actualPrimaryType": "unexplained_change", "actualReasoningModes": [ "establish_baseline", "identify_difference" ], "errors": [] }, { "id": "tc-016", "input": "The number of active users increased by 500%, from 4 to 2,001.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "unexplained_change", "actualReasoningModes": [ "identify_difference", "establish_baseline", "validate_measurement" ], "errors": [] }, { "id": "tc-017", "input": "User engagement metrics improved but the support ticket backlog grew by 200%.", "pass": false, "schemaValid": true, "classificationMatch": false, "reasoningModeMatch": true, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "unexplained_change", "actualReasoningModes": [ "identify_difference", "establish_baseline", "validate_measurement" ], "errors": [] }, { "id": "tc-018", "input": "The manufacturing team needs better quality control.", "pass": false, "schemaValid": true, "classificationMatch": false, "reasoningModeMatch": false, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "errors": [] }, { "id": "tc-019", "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "errors": [] }, { "id": "tc-020", "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "errors": [] }, { "id": "tc-021", "input": "Production output was 1,200 units last month and 1,180 units this month.", "pass": false, "schemaValid": true, "classificationMatch": false, "reasoningModeMatch": false, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 1, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "errors": [] }, { "id": "tc-022", "input": "The CFO reported that the company's cash position is healthy.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "reported_claim", "actualReasoningModes": [ "identify_difference", "validate_claim" ], "errors": [] }, { "id": "tc-023", "input": "We have enough funding to operate for 18 months.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "errors": [] }, { "id": "tc-024", "input": "The new feature was deployed at 3am and user complaints tripled the next day.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 1, "actualPrimaryType": "causal_claim", "actualReasoningModes": [ "establish_baseline", "identify_difference" ], "errors": [] }, { "id": "tc-025", "input": "We need to launch a mobile app to capture market share.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "decision_request", "actualReasoningModes": [ "identify_difference", "decision_support", "identify_missing_information" ], "errors": [] }, { "id": "tc-026", "input": "The system has been running for 90 days without failure since the migration.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 1, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "errors": [] }, { "id": "tc-027", "input": "No one has submitted the required compliance report despite multiple reminders.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "errors": [] }, { "id": "tc-028", "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "errors": [] }, { "id": "tc-029", "input": "We should implement the new CRM because our competitors have one.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 1, "actualPrimaryType": "decision_request", "actualReasoningModes": [ "identify_difference", "decision_support", "identify_missing_information" ], "errors": [] }, { "id": "tc-030", "input": "The server response time was acceptable last quarter but degraded this month.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "errors": [] }, { "id": "tc-031", "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "errors": [] }, { "id": "tc-032", "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.", "pass": false, "schemaValid": true, "classificationMatch": false, "reasoningModeMatch": false, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 1, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "errors": [] }, { "id": "tc-033", "input": "The database schema was changed on Friday but the reports are still working.", "pass": false, "schemaValid": true, "classificationMatch": false, "reasoningModeMatch": false, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "errors": [] }, { "id": "tc-034", "input": "Some team members say the new process is better while others say it's slower.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": false, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "errors": [] }, { "id": "tc-035", "input": "The application works fine on Chrome but not on Safari.", "pass": true, "schemaValid": true, "classificationMatch": true, "reasoningModeMatch": true, "requiredConceptMatch": false, "unsupportedInferenceAbsence": true, "nextQuestionPresent": true, "responseDurationMs": 0, "actualPrimaryType": "observed_problem", "actualReasoningModes": [ "identify_difference" ], "errors": [] } ] }