diff --git a/.gitignore b/.gitignore index a898f55..dd5091d 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,8 @@ Thumbs.db npm-debug.log* yarn-debug.log* yarn-error.log* + +# Generated evaluation artifacts (regenerated each run) +evaluation-results/ +provider-debug-results/ +tests-results/ diff --git a/evaluation-results/2026-08-01T06-01-56/diag-01-result.json b/evaluation-results/2026-08-01T06-01-56/diag-01-result.json deleted file mode 100644 index 3f4a30a..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-01-result.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "id": "diag-01", - "description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.", - "input": "We've seen a spike in complaints from our warehouse team this month compared to last month.", - "responseDurationMs": 6, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "complaints", - "found": true - }, - { - "concept": "warehouse", - "found": false - }, - { - "concept": "baseline comparison", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "quality issue", - "absent": true - }, - { - "concept": "staff turnover", - "absent": true - }, - { - "concept": "training gap", - "absent": true - } - ] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-01-56/diag-01-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-01-summary.md deleted file mode 100644 index eb447be..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-01-summary.md +++ /dev/null @@ -1,21 +0,0 @@ -# Diagnostic Case: diag-01 - -Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues. - -## Input - -``` -We've seen a spike in complaints from our warehouse team this month compared to last month. -``` - -## Result - -- **Technical**: ✅ PASS (1/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: unexplained_change -- **Actual Reasoning Modes**: establish_baseline, identify_difference -- **Response Duration**: 6ms - -### Reasoning Quality Failures - -missing required concept(s): warehouse, baseline comparison diff --git a/evaluation-results/2026-08-01T06-01-56/diag-02-result.json b/evaluation-results/2026-08-01T06-01-56/diag-02-result.json deleted file mode 100644 index 5555945..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-02-result.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "id": "diag-02", - "description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.", - "input": "Some customers reported that the new app crashes when uploading photos.", - "responseDurationMs": 2, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "app crashes", - "found": false - }, - { - "concept": "photo upload", - "found": false - }, - { - "concept": "some customers", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "all users affected", - "absent": true - }, - { - "concept": "server-side bug", - "absent": true - }, - { - "concept": "Android only", - "absent": true - } - ] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-01-56/diag-02-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-02-summary.md deleted file mode 100644 index 0172a70..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-02-summary.md +++ /dev/null @@ -1,21 +0,0 @@ -# Diagnostic Case: diag-02 - -Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims. - -## Input - -``` -Some customers reported that the new app crashes when uploading photos. -``` - -## Result - -- **Technical**: ✅ PASS (1/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: observed_problem -- **Actual Reasoning Modes**: identify_difference -- **Response Duration**: 2ms - -### Reasoning Quality Failures - -missing required concept(s): app crashes, photo upload, some customers diff --git a/evaluation-results/2026-08-01T06-01-56/diag-03-result.json b/evaluation-results/2026-08-01T06-01-56/diag-03-result.json deleted file mode 100644 index e9c28ca..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-03-result.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "id": "diag-03", - "description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.", - "input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.", - "responseDurationMs": 1, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": false, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "sales decline", - "found": false - }, - { - "concept": "price increase", - "found": false - }, - { - "concept": "revenue increase", - "found": false - }, - { - "concept": "CFO report", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "price was set too high", - "absent": true - }, - { - "concept": "competitors gained market share", - "absent": true - }, - { - "concept": "revenue data is wrong", - "absent": true - } - ] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-01-56/diag-03-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-03-summary.md deleted file mode 100644 index 6439e4d..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-03-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-03 - -Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value. - -## Input - -``` -Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: causal_claim -- **Actual Reasoning Modes**: establish_baseline, identify_difference -- **Response Duration**: 1ms - -### Technical Failures - -classification mismatch - -### Reasoning Quality Failures - -missing required concept(s): sales decline, price increase, revenue increase, CFO report diff --git a/evaluation-results/2026-08-01T06-01-56/diag-04-result.json b/evaluation-results/2026-08-01T06-01-56/diag-04-result.json deleted file mode 100644 index c8b849a..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-04-result.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "id": "diag-04", - "description": "Decision request — forward-looking, needs missing info identification.", - "input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.", - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "marketplace app", - "found": false - }, - { - "concept": "Southeast Asia", - "found": false - }, - { - "concept": "competitor gap", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "this will definitely succeed", - "absent": true - }, - { - "concept": "we have the resources", - "absent": true - }, - { - "concept": "competitors are struggling", - "absent": true - } - ] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-01-56/diag-04-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-04-summary.md deleted file mode 100644 index de5a547..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-04-summary.md +++ /dev/null @@ -1,21 +0,0 @@ -# Diagnostic Case: diag-04 - -Decision request — forward-looking, needs missing info identification. - -## Input - -``` -We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting. -``` - -## Result - -- **Technical**: ✅ PASS (1/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: decision_request -- **Actual Reasoning Modes**: identify_difference, decision_support, identify_missing_information -- **Response Duration**: 0ms - -### Reasoning Quality Failures - -missing required concept(s): marketplace app, Southeast Asia, competitor gap diff --git a/evaluation-results/2026-08-01T06-01-56/diag-05-result.json b/evaluation-results/2026-08-01T06-01-56/diag-05-result.json deleted file mode 100644 index a1551fc..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-05-result.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "id": "diag-05", - "description": "Unexpected continuity — changed context but no outcome change.", - "input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.", - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "supplier change", - "found": false - }, - { - "concept": "three months ago", - "found": false - }, - { - "concept": "same defect rate", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "new supplier is worse", - "absent": true - }, - { - "concept": "old supplier was better", - "absent": true - }, - { - "concept": "quality process is broken", - "absent": true - } - ] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-01-56/diag-05-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-05-summary.md deleted file mode 100644 index 4d24ef7..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-05-summary.md +++ /dev/null @@ -1,21 +0,0 @@ -# Diagnostic Case: diag-05 - -Unexpected continuity — changed context but no outcome change. - -## Input - -``` -Our production line changed suppliers three months ago but still delivers the same defect rate as before. -``` - -## Result - -- **Technical**: ✅ PASS (1/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: unexplained_change -- **Actual Reasoning Modes**: identify_difference, establish_baseline, validate_measurement -- **Response Duration**: 1ms - -### Reasoning Quality Failures - -missing required concept(s): supplier change, three months ago, same defect rate diff --git a/evaluation-results/2026-08-01T06-01-56/diag-06-result.json b/evaluation-results/2026-08-01T06-01-56/diag-06-result.json deleted file mode 100644 index 2f6f608..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-06-result.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "id": "diag-06", - "description": "Quantified improvement — needs context about measurement period and baseline conditions.", - "input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.", - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": false, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "completion rate", - "found": false - }, - { - "concept": "45%", - "found": false - }, - { - "concept": "62%", - "found": false - }, - { - "concept": "onboarding", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "all improvements are due to the redesign", - "absent": true - }, - { - "concept": "the old flow was bad", - "absent": true - }, - { - "concept": "users prefer the new design", - "absent": true - } - ] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-01-56/diag-06-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-06-summary.md deleted file mode 100644 index b551743..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-06-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-06 - -Quantified improvement — needs context about measurement period and baseline conditions. - -## Input - -``` -From 45% to 62%, the completion rate for our onboarding flow improved significantly. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: observed_problem -- **Actual Reasoning Modes**: identify_difference -- **Response Duration**: 1ms - -### Technical Failures - -classification mismatch - -### Reasoning Quality Failures - -missing required concept(s): completion rate, 45%, 62%, onboarding diff --git a/evaluation-results/2026-08-01T06-01-56/diag-07-result.json b/evaluation-results/2026-08-01T06-01-56/diag-07-result.json deleted file mode 100644 index e902e9c..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-07-result.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "id": "diag-07", - "description": "Single reported claim — needs validation, not acceptance as fact.", - "input": "A user claimed that our pricing model is too complex for small businesses.", - "responseDurationMs": 1, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "pricing complexity", - "found": false - }, - { - "concept": "small business", - "found": false - }, - { - "concept": "user claim", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "the pricing is actually complex", - "absent": true - }, - { - "concept": "other small businesses agree", - "absent": true - }, - { - "concept": "we should simplify pricing", - "absent": true - } - ] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-01-56/diag-07-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-07-summary.md deleted file mode 100644 index 4a8a7d7..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-07-summary.md +++ /dev/null @@ -1,21 +0,0 @@ -# Diagnostic Case: diag-07 - -Single reported claim — needs validation, not acceptance as fact. - -## Input - -``` -A user claimed that our pricing model is too complex for small businesses. -``` - -## Result - -- **Technical**: ✅ PASS (1/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: reported_claim -- **Actual Reasoning Modes**: identify_difference, validate_claim -- **Response Duration**: 1ms - -### Reasoning Quality Failures - -missing required concept(s): pricing complexity, small business, user claim diff --git a/evaluation-results/2026-08-01T06-01-56/diag-08-result.json b/evaluation-results/2026-08-01T06-01-56/diag-08-result.json deleted file mode 100644 index bdbef7d..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-08-result.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "id": "diag-08", - "description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode.", - "input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?", - "responseDurationMs": 1, - "actualPrimaryType": "ambiguous_statement", - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "philosophical", - "found": false - }, - { - "concept": "ambiguous", - "found": true - }, - { - "concept": "meaning clarification", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "the phrase was wrong", - "absent": true - }, - { - "concept": "the colleague is hostile", - "absent": true - }, - { - "concept": "we should avoid philosophical language", - "absent": true - } - ] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-01-56/diag-08-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-08-summary.md deleted file mode 100644 index b5cd5a8..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-08-summary.md +++ /dev/null @@ -1,21 +0,0 @@ -# Diagnostic Case: diag-08 - -Meta-test — self-referential ambiguous statement. Should trigger clarification mode. - -## Input - -``` -I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair? -``` - -## Result - -- **Technical**: ✅ PASS (1/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: ambiguous_statement -- **Actual Reasoning Modes**: identify_difference, clarify_meaning -- **Response Duration**: 1ms - -### Reasoning Quality Failures - -missing required concept(s): philosophical, meaning clarification diff --git a/evaluation-results/2026-08-01T06-01-56/diag-09-result.json b/evaluation-results/2026-08-01T06-01-56/diag-09-result.json deleted file mode 100644 index 367933f..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-09-result.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "id": "diag-09", - "description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.", - "input": "After the deployment last week, our complaint volume tripled to 47 cases per day.", - "responseDurationMs": 1, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "deployment", - "found": false - }, - { - "concept": "complaint volume increase", - "found": false - }, - { - "concept": "tripled", - "found": false - }, - { - "concept": "47 cases", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "the deployment caused the complaints", - "absent": true - }, - { - "concept": "the bug report was insufficient", - "absent": true - }, - { - "concept": "rollback is needed", - "absent": true - } - ] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-01-56/diag-09-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-09-summary.md deleted file mode 100644 index a9a517f..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-09-summary.md +++ /dev/null @@ -1,21 +0,0 @@ -# Diagnostic Case: diag-09 - -Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion. - -## Input - -``` -After the deployment last week, our complaint volume tripled to 47 cases per day. -``` - -## Result - -- **Technical**: ✅ PASS (1/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: causal_claim -- **Actual Reasoning Modes**: establish_baseline, identify_difference -- **Response Duration**: 1ms - -### Reasoning Quality Failures - -missing required concept(s): deployment, complaint volume increase, tripled, 47 cases diff --git a/evaluation-results/2026-08-01T06-01-56/diag-10-result.json b/evaluation-results/2026-08-01T06-01-56/diag-10-result.json deleted file mode 100644 index 5e1fcab..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-10-result.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "id": "diag-10", - "description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims.", - "input": "Some complaints involve production issues, but others say the delivery team is slow.", - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "production issues", - "found": false - }, - { - "concept": "delivery speed", - "found": false - }, - { - "concept": "complaint types", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "production is worse than delivery", - "absent": true - }, - { - "concept": "the delivery team needs training", - "absent": true - }, - { - "concept": "both teams are underperforming equally", - "absent": true - } - ] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-01-56/diag-10-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-10-summary.md deleted file mode 100644 index ad5a73c..0000000 --- a/evaluation-results/2026-08-01T06-01-56/diag-10-summary.md +++ /dev/null @@ -1,21 +0,0 @@ -# Diagnostic Case: diag-10 - -Paired with diag-01 — distinguishes subset complaints from aggregate claims. - -## Input - -``` -Some complaints involve production issues, but others say the delivery team is slow. -``` - -## Result - -- **Technical**: ✅ PASS (1/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: observed_problem -- **Actual Reasoning Modes**: establish_baseline, identify_difference -- **Response Duration**: 1ms - -### Reasoning Quality Failures - -missing required concept(s): production issues, delivery speed, complaint types diff --git a/evaluation-results/2026-08-01T06-01-56/summary.json b/evaluation-results/2026-08-01T06-01-56/summary.json deleted file mode 100644 index d0bde97..0000000 --- a/evaluation-results/2026-08-01T06-01-56/summary.json +++ /dev/null @@ -1,585 +0,0 @@ -{ - "timestamp": "2026-08-01T06:01:56.070Z", - "provider": "mock", - "promptVersion": "v0.2", - "casesRun": 10, - "summary": { - "technical": { - "schemaValidityRate": "100.0%", - "classificationMatchRate": "80.0%", - "nextQuestionPresentRate": "100.0%", - "passRate": "80.0%" - }, - "reasoningQuality": { - "requiredConceptMatchRate": "0.0%", - "unsupportedInferenceFailures": "0", - "passRate": "0.0%" - }, - "combinedPassRate": "0.0%", - "averageResponseDurationMs": "2" - }, - "testCaseResults": [ - { - "id": "diag-01", - "input": "We've seen a spike in complaints from our warehouse team this month compared to last month.", - "responseDurationMs": 6, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "complaints", - "found": true - }, - { - "concept": "warehouse", - "found": false - }, - { - "concept": "baseline comparison", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "quality issue", - "absent": true - }, - { - "concept": "staff turnover", - "absent": true - }, - { - "concept": "training gap", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "diag-02", - "input": "Some customers reported that the new app crashes when uploading photos.", - "responseDurationMs": 2, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "app crashes", - "found": false - }, - { - "concept": "photo upload", - "found": false - }, - { - "concept": "some customers", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "all users affected", - "absent": true - }, - { - "concept": "server-side bug", - "absent": true - }, - { - "concept": "Android only", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "diag-03", - "input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.", - "responseDurationMs": 1, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": false, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "sales decline", - "found": false - }, - { - "concept": "price increase", - "found": false - }, - { - "concept": "revenue increase", - "found": false - }, - { - "concept": "CFO report", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "price was set too high", - "absent": true - }, - { - "concept": "competitors gained market share", - "absent": true - }, - { - "concept": "revenue data is wrong", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "diag-04", - "input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.", - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "marketplace app", - "found": false - }, - { - "concept": "Southeast Asia", - "found": false - }, - { - "concept": "competitor gap", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "this will definitely succeed", - "absent": true - }, - { - "concept": "we have the resources", - "absent": true - }, - { - "concept": "competitors are struggling", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "diag-05", - "input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.", - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "supplier change", - "found": false - }, - { - "concept": "three months ago", - "found": false - }, - { - "concept": "same defect rate", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "new supplier is worse", - "absent": true - }, - { - "concept": "old supplier was better", - "absent": true - }, - { - "concept": "quality process is broken", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "diag-06", - "input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.", - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": false, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "completion rate", - "found": false - }, - { - "concept": "45%", - "found": false - }, - { - "concept": "62%", - "found": false - }, - { - "concept": "onboarding", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "all improvements are due to the redesign", - "absent": true - }, - { - "concept": "the old flow was bad", - "absent": true - }, - { - "concept": "users prefer the new design", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "diag-07", - "input": "A user claimed that our pricing model is too complex for small businesses.", - "responseDurationMs": 1, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "pricing complexity", - "found": false - }, - { - "concept": "small business", - "found": false - }, - { - "concept": "user claim", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "the pricing is actually complex", - "absent": true - }, - { - "concept": "other small businesses agree", - "absent": true - }, - { - "concept": "we should simplify pricing", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "diag-08", - "input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?", - "responseDurationMs": 1, - "actualPrimaryType": "ambiguous_statement", - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "philosophical", - "found": false - }, - { - "concept": "ambiguous", - "found": true - }, - { - "concept": "meaning clarification", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "the phrase was wrong", - "absent": true - }, - { - "concept": "the colleague is hostile", - "absent": true - }, - { - "concept": "we should avoid philosophical language", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "diag-09", - "input": "After the deployment last week, our complaint volume tripled to 47 cases per day.", - "responseDurationMs": 1, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "deployment", - "found": false - }, - { - "concept": "complaint volume increase", - "found": false - }, - { - "concept": "tripled", - "found": false - }, - { - "concept": "47 cases", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "the deployment caused the complaints", - "absent": true - }, - { - "concept": "the bug report was insufficient", - "absent": true - }, - { - "concept": "rollback is needed", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "diag-10", - "input": "Some complaints involve production issues, but others say the delivery team is slow.", - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "production issues", - "found": false - }, - { - "concept": "delivery speed", - "found": false - }, - { - "concept": "complaint types", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "production is worse than delivery", - "absent": true - }, - { - "concept": "the delivery team needs training", - "absent": true - }, - { - "concept": "both teams are underperforming equally", - "absent": true - } - ] - }, - "pass": false - } - } - ] -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-07-42/diag-01-result.json b/evaluation-results/2026-08-01T06-07-42/diag-01-result.json deleted file mode 100644 index 5e47ef7..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-01-result.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "id": "diag-01", - "description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.", - "input": "We've seen a spike in complaints from our warehouse team this month compared to last month.", - "responseDurationMs": 1785564462515, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-07-42/diag-01-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-01-summary.md deleted file mode 100644 index 1570f54..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-01-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-01 - -Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues. - -## Input - -``` -We've seen a spike in complaints from our warehouse team this month compared to last month. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 1785564462515ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-07-42/diag-02-result.json b/evaluation-results/2026-08-01T06-07-42/diag-02-result.json deleted file mode 100644 index 1e9e5ed..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-02-result.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "id": "diag-02", - "description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.", - "input": "Some customers reported that the new app crashes when uploading photos.", - "responseDurationMs": 1785564462519, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-07-42/diag-02-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-02-summary.md deleted file mode 100644 index 0518536..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-02-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-02 - -Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims. - -## Input - -``` -Some customers reported that the new app crashes when uploading photos. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 1785564462519ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-07-42/diag-03-result.json b/evaluation-results/2026-08-01T06-07-42/diag-03-result.json deleted file mode 100644 index 4d30197..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-03-result.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "id": "diag-03", - "description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.", - "input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.", - "responseDurationMs": 1785564462519, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-07-42/diag-03-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-03-summary.md deleted file mode 100644 index 7180883..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-03-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-03 - -Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value. - -## Input - -``` -Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 1785564462519ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-07-42/diag-04-result.json b/evaluation-results/2026-08-01T06-07-42/diag-04-result.json deleted file mode 100644 index d62c283..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-04-result.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "id": "diag-04", - "description": "Decision request — forward-looking, needs missing info identification.", - "input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.", - "responseDurationMs": 1785564462520, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-07-42/diag-04-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-04-summary.md deleted file mode 100644 index 7229c5d..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-04-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-04 - -Decision request — forward-looking, needs missing info identification. - -## Input - -``` -We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 1785564462520ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-07-42/diag-05-result.json b/evaluation-results/2026-08-01T06-07-42/diag-05-result.json deleted file mode 100644 index f957f54..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-05-result.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "id": "diag-05", - "description": "Unexpected continuity — changed context but no outcome change.", - "input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.", - "responseDurationMs": 1785564462520, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-07-42/diag-05-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-05-summary.md deleted file mode 100644 index 0389218..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-05-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-05 - -Unexpected continuity — changed context but no outcome change. - -## Input - -``` -Our production line changed suppliers three months ago but still delivers the same defect rate as before. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 1785564462520ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-07-42/diag-06-result.json b/evaluation-results/2026-08-01T06-07-42/diag-06-result.json deleted file mode 100644 index f4255c3..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-06-result.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "id": "diag-06", - "description": "Quantified improvement — needs context about measurement period and baseline conditions.", - "input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.", - "responseDurationMs": 1785564462521, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-07-42/diag-06-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-06-summary.md deleted file mode 100644 index 02da337..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-06-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-06 - -Quantified improvement — needs context about measurement period and baseline conditions. - -## Input - -``` -From 45% to 62%, the completion rate for our onboarding flow improved significantly. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 1785564462521ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-07-42/diag-07-result.json b/evaluation-results/2026-08-01T06-07-42/diag-07-result.json deleted file mode 100644 index fcdb417..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-07-result.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "id": "diag-07", - "description": "Single reported claim — needs validation, not acceptance as fact.", - "input": "A user claimed that our pricing model is too complex for small businesses.", - "responseDurationMs": 1785564462521, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-07-42/diag-07-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-07-summary.md deleted file mode 100644 index 54e004f..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-07-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-07 - -Single reported claim — needs validation, not acceptance as fact. - -## Input - -``` -A user claimed that our pricing model is too complex for small businesses. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 1785564462521ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-07-42/diag-08-result.json b/evaluation-results/2026-08-01T06-07-42/diag-08-result.json deleted file mode 100644 index e07f754..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-08-result.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "id": "diag-08", - "description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode.", - "input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?", - "responseDurationMs": 1785564462521, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-07-42/diag-08-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-08-summary.md deleted file mode 100644 index ff01aa7..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-08-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-08 - -Meta-test — self-referential ambiguous statement. Should trigger clarification mode. - -## Input - -``` -I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair? -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 1785564462521ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-07-42/diag-09-result.json b/evaluation-results/2026-08-01T06-07-42/diag-09-result.json deleted file mode 100644 index 75a0fce..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-09-result.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "id": "diag-09", - "description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.", - "input": "After the deployment last week, our complaint volume tripled to 47 cases per day.", - "responseDurationMs": 1785564462521, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-07-42/diag-09-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-09-summary.md deleted file mode 100644 index 7dafaba..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-09-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-09 - -Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion. - -## Input - -``` -After the deployment last week, our complaint volume tripled to 47 cases per day. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 1785564462521ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-07-42/diag-10-result.json b/evaluation-results/2026-08-01T06-07-42/diag-10-result.json deleted file mode 100644 index 7f1de1f..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-10-result.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "id": "diag-10", - "description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims.", - "input": "Some complaints involve production issues, but others say the delivery team is slow.", - "responseDurationMs": 1785564462522, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-07-42/diag-10-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-10-summary.md deleted file mode 100644 index 9e20b0b..0000000 --- a/evaluation-results/2026-08-01T06-07-42/diag-10-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-10 - -Paired with diag-01 — distinguishes subset complaints from aggregate claims. - -## Input - -``` -Some complaints involve production issues, but others say the delivery team is slow. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 1785564462522ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-07-42/summary.json b/evaluation-results/2026-08-01T06-07-42/summary.json deleted file mode 100644 index 37bdef7..0000000 --- a/evaluation-results/2026-08-01T06-07-42/summary.json +++ /dev/null @@ -1,313 +0,0 @@ -{ - "timestamp": "2026-08-01T06:07:42.532Z", - "provider": "ollama-real", - "promptVersion": "v0.2", - "casesRun": 10, - "summary": { - "technical": { - "schemaValidityRate": "0.0%", - "classificationMatchRate": "0.0%", - "nextQuestionPresentRate": "0.0%", - "passRate": "0.0%" - }, - "reasoningQuality": { - "requiredConceptMatchRate": "100.0%", - "unsupportedInferenceFailures": "0", - "passRate": "0.0%" - }, - "combinedPassRate": "0.0%", - "averageResponseDurationMs": "1785564462520" - }, - "testCaseResults": [ - { - "id": "diag-01", - "input": "We've seen a spike in complaints from our warehouse team this month compared to last month.", - "responseDurationMs": 1785564462515, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "diag-02", - "input": "Some customers reported that the new app crashes when uploading photos.", - "responseDurationMs": 1785564462519, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "diag-03", - "input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.", - "responseDurationMs": 1785564462519, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "diag-04", - "input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.", - "responseDurationMs": 1785564462520, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "diag-05", - "input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.", - "responseDurationMs": 1785564462520, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "diag-06", - "input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.", - "responseDurationMs": 1785564462521, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "diag-07", - "input": "A user claimed that our pricing model is too complex for small businesses.", - "responseDurationMs": 1785564462521, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "diag-08", - "input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?", - "responseDurationMs": 1785564462521, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "diag-09", - "input": "After the deployment last week, our complaint volume tripled to 47 cases per day.", - "responseDurationMs": 1785564462521, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "diag-10", - "input": "Some complaints involve production issues, but others say the delivery team is slow.", - "responseDurationMs": 1785564462522, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Invalid server configuration", - "Invalid server configuration" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - } - ] -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-11-41/diag-01-result.json b/evaluation-results/2026-08-01T06-11-41/diag-01-result.json deleted file mode 100644 index 0ceb523..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-01-result.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "id": "diag-01", - "description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.", - "input": "We've seen a spike in complaints from our warehouse team this month compared to last month.", - "responseDurationMs": 19459, - "actualPrimaryType": null, - "actualReasoningModes": [], - "rawOutput": "{\"status\":\"received\",\"message\":\"Please provide a specific request or data to process.\"}", - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "inputClassification: Required", - "reconstruction: Required", - "evidence: Required", - "nextQuestion: Required" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-11-41/diag-01-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-01-summary.md deleted file mode 100644 index c68eeac..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-01-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-01 - -Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues. - -## Input - -``` -We've seen a spike in complaints from our warehouse team this month compared to last month. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 19459ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-11-41/diag-02-result.json b/evaluation-results/2026-08-01T06-11-41/diag-02-result.json deleted file mode 100644 index 0120dce..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-02-result.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "id": "diag-02", - "description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.", - "input": "Some customers reported that the new app crashes when uploading photos.", - "responseDurationMs": 14693, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download", - "Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-11-41/diag-02-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-02-summary.md deleted file mode 100644 index 706b324..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-02-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-02 - -Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims. - -## Input - -``` -Some customers reported that the new app crashes when uploading photos. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 14693ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-11-41/diag-03-result.json b/evaluation-results/2026-08-01T06-11-41/diag-03-result.json deleted file mode 100644 index 3aa6882..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-03-result.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "id": "diag-03", - "description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.", - "input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.", - "responseDurationMs": 13516, - "actualPrimaryType": null, - "actualReasoningModes": [], - "rawOutput": "{\"status\":\"success\",\"message\":\"Input received and processed.\"}", - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "inputClassification: Required", - "reconstruction: Required", - "evidence: Required", - "nextQuestion: Required" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-11-41/diag-03-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-03-summary.md deleted file mode 100644 index 0a4c7f3..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-03-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-03 - -Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value. - -## Input - -``` -Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 13516ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-11-41/diag-04-result.json b/evaluation-results/2026-08-01T06-11-41/diag-04-result.json deleted file mode 100644 index 881b637..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-04-result.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "id": "diag-04", - "description": "Decision request — forward-looking, needs missing info identification.", - "input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.", - "responseDurationMs": 48851, - "actualPrimaryType": null, - "actualReasoningModes": [], - "rawOutput": "{\"state\":\"pending\"}", - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "inputClassification: Required", - "reconstruction: Required", - "evidence: Required", - "nextQuestion: Required" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-11-41/diag-04-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-04-summary.md deleted file mode 100644 index dcf3fe9..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-04-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-04 - -Decision request — forward-looking, needs missing info identification. - -## Input - -``` -We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 48851ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-11-41/diag-05-result.json b/evaluation-results/2026-08-01T06-11-41/diag-05-result.json deleted file mode 100644 index 2f5064d..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-05-result.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "id": "diag-05", - "description": "Unexpected continuity — changed context but no outcome change.", - "input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.", - "responseDurationMs": 16508, - "actualPrimaryType": null, - "actualReasoningModes": [], - "rawOutput": "{}", - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "inputClassification: Required", - "reconstruction: Required", - "evidence: Required", - "nextQuestion: Required" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-11-41/diag-05-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-05-summary.md deleted file mode 100644 index 4a87c63..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-05-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-05 - -Unexpected continuity — changed context but no outcome change. - -## Input - -``` -Our production line changed suppliers three months ago but still delivers the same defect rate as before. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 16508ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-11-41/diag-06-result.json b/evaluation-results/2026-08-01T06-11-41/diag-06-result.json deleted file mode 100644 index 923a8c9..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-06-result.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "id": "diag-06", - "description": "Quantified improvement — needs context about measurement period and baseline conditions.", - "input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.", - "responseDurationMs": 15608, - "actualPrimaryType": null, - "actualReasoningModes": [], - "rawOutput": "{\"status\":\"ok\"}", - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "inputClassification: Required", - "reconstruction: Required", - "evidence: Required", - "nextQuestion: Required" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-11-41/diag-06-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-06-summary.md deleted file mode 100644 index 0a02208..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-06-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-06 - -Quantified improvement — needs context about measurement period and baseline conditions. - -## Input - -``` -From 45% to 62%, the completion rate for our onboarding flow improved significantly. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 15608ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-11-41/diag-07-result.json b/evaluation-results/2026-08-01T06-11-41/diag-07-result.json deleted file mode 100644 index 2476ec3..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-07-result.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "id": "diag-07", - "description": "Single reported claim — needs validation, not acceptance as fact.", - "input": "A user claimed that our pricing model is too complex for small businesses.", - "responseDurationMs": 15750, - "actualPrimaryType": null, - "actualReasoningModes": [], - "rawOutput": "{\"status\":\"success\",\"message\":\"Input received\"}", - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "inputClassification: Required", - "reconstruction: Required", - "evidence: Required", - "nextQuestion: Required" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-11-41/diag-07-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-07-summary.md deleted file mode 100644 index cbd3515..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-07-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-07 - -Single reported claim — needs validation, not acceptance as fact. - -## Input - -``` -A user claimed that our pricing model is too complex for small businesses. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 15750ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-11-41/diag-08-result.json b/evaluation-results/2026-08-01T06-11-41/diag-08-result.json deleted file mode 100644 index 9ce18ae..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-08-result.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "id": "diag-08", - "description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode.", - "input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?", - "responseDurationMs": 15383, - "actualPrimaryType": null, - "actualReasoningModes": [], - "rawOutput": "{\"status\":\"success\",\"message\":\"JSON object returned as requested\"}", - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "inputClassification: Required", - "reconstruction: Required", - "evidence: Required", - "nextQuestion: Required" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-11-41/diag-08-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-08-summary.md deleted file mode 100644 index fed9f98..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-08-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-08 - -Meta-test — self-referential ambiguous statement. Should trigger clarification mode. - -## Input - -``` -I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair? -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 15383ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-11-41/diag-09-result.json b/evaluation-results/2026-08-01T06-11-41/diag-09-result.json deleted file mode 100644 index f80452c..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-09-result.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "id": "diag-09", - "description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.", - "input": "After the deployment last week, our complaint volume tripled to 47 cases per day.", - "responseDurationMs": 15643, - "actualPrimaryType": null, - "actualReasoningModes": [], - "rawOutput": "{\"status\":\"ok\"}", - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "inputClassification: Required", - "reconstruction: Required", - "evidence: Required", - "nextQuestion: Required" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-11-41/diag-09-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-09-summary.md deleted file mode 100644 index fb68c09..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-09-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-09 - -Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion. - -## Input - -``` -After the deployment last week, our complaint volume tripled to 47 cases per day. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 15643ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-11-41/diag-10-result.json b/evaluation-results/2026-08-01T06-11-41/diag-10-result.json deleted file mode 100644 index 9f59f16..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-10-result.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "id": "diag-10", - "description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims.", - "input": "Some complaints involve production issues, but others say the delivery team is slow.", - "responseDurationMs": 15401, - "actualPrimaryType": null, - "actualReasoningModes": [], - "rawOutput": "{}", - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "inputClassification: Required", - "reconstruction: Required", - "evidence: Required", - "nextQuestion: Required" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } -} \ No newline at end of file diff --git a/evaluation-results/2026-08-01T06-11-41/diag-10-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-10-summary.md deleted file mode 100644 index 3067f33..0000000 --- a/evaluation-results/2026-08-01T06-11-41/diag-10-summary.md +++ /dev/null @@ -1,25 +0,0 @@ -# Diagnostic Case: diag-10 - -Paired with diag-01 — distinguishes subset complaints from aggregate claims. - -## Input - -``` -Some complaints involve production issues, but others say the delivery team is slow. -``` - -## Result - -- **Technical**: ❌ FAIL (0/4 sub-checks pass) -- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass) -- **Actual Primary Type**: N/A -- **Actual Reasoning Modes**: N/A -- **Response Duration**: 15401ms - -### Technical Failures - -schema invalid, classification mismatch, no next question - -### Reasoning Quality Failures - - diff --git a/evaluation-results/2026-08-01T06-11-41/summary.json b/evaluation-results/2026-08-01T06-11-41/summary.json deleted file mode 100644 index 3e65eac..0000000 --- a/evaluation-results/2026-08-01T06-11-41/summary.json +++ /dev/null @@ -1,331 +0,0 @@ -{ - "timestamp": "2026-08-01T06:11:41.538Z", - "provider": "ollama-real", - "promptVersion": "v0.2", - "casesRun": 10, - "summary": { - "technical": { - "schemaValidityRate": "0.0%", - "classificationMatchRate": "0.0%", - "nextQuestionPresentRate": "0.0%", - "passRate": "0.0%" - }, - "reasoningQuality": { - "requiredConceptMatchRate": "100.0%", - "unsupportedInferenceFailures": "0", - "passRate": "0.0%" - }, - "combinedPassRate": "0.0%", - "averageResponseDurationMs": "19081" - }, - "testCaseResults": [ - { - "id": "diag-01", - "input": "We've seen a spike in complaints from our warehouse team this month compared to last month.", - "responseDurationMs": 19459, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "inputClassification: Required", - "reconstruction: Required", - "evidence: Required", - "nextQuestion: Required" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "diag-02", - "input": "Some customers reported that the new app crashes when uploading photos.", - "responseDurationMs": 14693, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download", - "Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "diag-03", - "input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.", - "responseDurationMs": 13516, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "inputClassification: Required", - "reconstruction: Required", - "evidence: Required", - "nextQuestion: Required" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "diag-04", - "input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.", - "responseDurationMs": 48851, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "inputClassification: Required", - "reconstruction: Required", - "evidence: Required", - "nextQuestion: Required" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "diag-05", - "input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.", - "responseDurationMs": 16508, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "inputClassification: Required", - "reconstruction: Required", - "evidence: Required", - "nextQuestion: Required" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "diag-06", - "input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.", - "responseDurationMs": 15608, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "inputClassification: Required", - "reconstruction: Required", - "evidence: Required", - "nextQuestion: Required" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "diag-07", - "input": "A user claimed that our pricing model is too complex for small businesses.", - "responseDurationMs": 15750, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "inputClassification: Required", - "reconstruction: Required", - "evidence: Required", - "nextQuestion: Required" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "diag-08", - "input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?", - "responseDurationMs": 15383, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "inputClassification: Required", - "reconstruction: Required", - "evidence: Required", - "nextQuestion: Required" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "diag-09", - "input": "After the deployment last week, our complaint volume tripled to 47 cases per day.", - "responseDurationMs": 15643, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "inputClassification: Required", - "reconstruction: Required", - "evidence: Required", - "nextQuestion: Required" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "diag-10", - "input": "Some complaints involve production issues, but others say the delivery team is slow.", - "responseDurationMs": 15401, - "actualPrimaryType": null, - "actualReasoningModes": [], - "technical": { - "schemaValid": false, - "classificationMatch": false, - "reasoningModeMatch": false, - "nextQuestionPresent": false, - "pass": false, - "errors": [ - "inputClassification: Required", - "reconstruction: Required", - "evidence: Required", - "nextQuestion: Required" - ] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": true, - "details": [] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - } - ] -} \ No newline at end of file diff --git a/evaluation-results/latest-manifest.json b/evaluation-results/latest-manifest.json deleted file mode 100644 index 18f27f5..0000000 --- a/evaluation-results/latest-manifest.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "latestRun": "2026-08-01T06-11-41", - "caseCount": 10 -} \ No newline at end of file diff --git a/provider-debug-results/2026-08-01T06-35-55.897Z/debug-results.json b/provider-debug-results/2026-08-01T06-35-55.897Z/debug-results.json deleted file mode 100644 index a19f9f7..0000000 --- a/provider-debug-results/2026-08-01T06-35-55.897Z/debug-results.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "qwen-claude:latest": { - "availability": "NOT_AVAILABLE_ON_SERVER" - }, - "qwen3.6:35b-a3b": { - "availability": "NOT_AVAILABLE_ON_SERVER" - } -} \ No newline at end of file diff --git a/provider-debug-results/2026-08-01T06-37-17.829Z/debug-results.json b/provider-debug-results/2026-08-01T06-37-17.829Z/debug-results.json deleted file mode 100644 index 5b11bb7..0000000 --- a/provider-debug-results/2026-08-01T06-37-17.829Z/debug-results.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "qwen-claude:latest": { - "test_A_qwen_claude": { - "testDescription": "Plain instruction test — should return CHAT_WORKS", - "endpoint": "/api/chat", - "format": "json", - "hasSystemMessage": true, - "httpStatus": 200, - "messageContentType": "string", - "messageContentLength": 30, - "thinkingPresent": true, - "parsedContentKeys": [ - "response" - ], - "looksLikeStatusAck": false, - "rawPreview": "{\n \"response\": \"CHAT_WORKS\"\n}" - }, - "test_A_qwen_claude_generate": { - "endpoint": "/api/generate", - "httpStatus": 200, - "responseFirst200": "CHAT_WORKS", - "responseLooksLikeStructuredJSON": false, - "rawPreview": "CHAT_WORKS" - }, - "test_B_qwen_claude": { - "testDescription": "JSON schema test — should return exact object", - "endpoint": "/api/chat", - "format": "json", - "hasSystemMessage": false, - "httpStatus": 200, - "messageContentType": "string", - "messageContentLength": 38, - "thinkingPresent": true, - "parsedContentKeys": [ - "message" - ], - "looksLikeStatusAck": false, - "rawPreview": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}" - }, - "test_B_qwen_claude_generate": { - "endpoint": "/api/generate", - "httpStatus": 200, - "responseFirst200": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}", - "responseLooksLikeStructuredJSON": true, - "rawPreview": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}" - }, - "test_C_qwen_claude": { - "testDescription": "Minimal reconstruction schema — structured output test", - "endpoint": "/api/chat", - "format": "json", - "hasSystemMessage": false, - "httpStatus": 200, - "messageContentType": "string", - "messageContentLength": 1145, - "thinkingPresent": true, - "parsedContentKeys": [ - "analysis", - "meaningful_difference", - "next_question" - ], - "looksLikeStatusAck": false, - "rawPreview": "{\"analysis\":\"The workflow breaks into two distinct stages: authentication (login) and resource access (invoice download). Since login succeeds for the affected users, identity verification and session creation are functioning correctly. The failure occurs downstream in the layer responsible for loca" - }, - "test_C_qwen_claude_generate": { - "endpoint": "/api/generate", - "httpStatus": 200, - "responseFirst200": "**Analysis of the Situation**\nThe issue represents a partial, post-authentication failure: identity verification works for a subset of users, but access to or delivery of a specific resource (invoices", - "responseLooksLikeStructuredJSON": false, - "rawPreview": "**Analysis of the Situation**\nThe issue represents a partial, post-authentication failure: identity verification works for a subset of users, but access to or delivery of a specific resource (invoices" - } - }, - "qwen3.6:35b-a3b": { - "test_A_qwen___": { - "testDescription": "Plain instruction test — should return CHAT_WORKS", - "endpoint": "/api/chat", - "format": "json", - "hasSystemMessage": true, - "httpStatus": 200, - "messageContentType": "string", - "messageContentLength": 30, - "thinkingPresent": true, - "parsedContentKeys": [ - "response" - ], - "looksLikeStatusAck": false, - "rawPreview": "{\n \"response\": \"CHAT_WORKS\"\n}" - }, - "test_A_qwen____generate": { - "endpoint": "/api/generate", - "httpStatus": 200, - "responseFirst200": "CHAT_WORKS", - "responseLooksLikeStructuredJSON": false, - "rawPreview": "CHAT_WORKS" - }, - "test_B_qwen___": { - "testDescription": "JSON schema test — should return exact object", - "endpoint": "/api/chat", - "format": "json", - "hasSystemMessage": false, - "httpStatus": 200, - "messageContentType": "string", - "messageContentLength": 38, - "thinkingPresent": true, - "parsedContentKeys": [ - "message" - ], - "looksLikeStatusAck": false, - "rawPreview": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}" - }, - "test_B_qwen____generate": { - "endpoint": "/api/generate", - "httpStatus": 200, - "responseFirst200": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}", - "responseLooksLikeStructuredJSON": true, - "rawPreview": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}" - }, - "test_C_qwen___": { - "testDescription": "Minimal reconstruction schema — structured output test", - "endpoint": "/api/chat", - "format": "json", - "hasSystemMessage": false, - "httpStatus": 200, - "messageContentType": "string", - "messageContentLength": 843, - "thinkingPresent": true, - "parsedContentKeys": [ - "analysis", - "meaningful_difference", - "next_question" - ], - "looksLikeStatusAck": false, - "rawPreview": "{\"analysis\": \"The scenario indicates that authentication is functioning correctly, but a downstream access or resource-retrieval step is failing for a subset of users. This separates the problem from credential or session validity and points toward post-login factors such as authorization scope, doc" - }, - "test_C_qwen____generate": { - "endpoint": "/api/generate", - "httpStatus": 200, - "responseFirst200": "**Meaningful Difference:** \nSuccessful login verifies *account-level authentication and baseline platform access*, while downloading an invoice requires *invoice-specific data availability, retrieval", - "responseLooksLikeStructuredJSON": false, - "rawPreview": "**Meaningful Difference:** \nSuccessful login verifies *account-level authentication and baseline platform access*, while downloading an invoice requires *invoice-specific data availability, retrieval" - } - } -} \ No newline at end of file diff --git a/tests-results/evaluation-2026-07-31T18-14-22.json b/tests-results/evaluation-2026-07-31T18-14-22.json deleted file mode 100644 index 416f672..0000000 --- a/tests-results/evaluation-2026-07-31T18-14-22.json +++ /dev/null @@ -1,653 +0,0 @@ -{ - "timestamp": "2026-07-31T18:14:22.632Z", - "provider": "mock", - "promptVersion": "v0.2", - "casesRun": 35, - "summary": { - "schemaValidityRate": "100.0%", - "classificationMatchRate": "28.6%", - "reasoningModeMatchRate": "48.6%", - "requiredConceptMatchRate": "0.0%", - "unsupportedInferenceFailures": "0", - "averageResponseDurationMs": "1", - "fullPassRate": "28.6%" - }, - "testCaseResults": [ - { - "id": "tc-001", - "input": "All customers cannot download their invoices.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 4, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-002", - "input": "Some customers cannot download their invoices.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-003", - "input": "Complaints increased by 35%.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-004", - "input": "Complaints increased by 35% while production increased by 40%.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-005", - "input": "Sales are falling.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-006", - "input": "Sales fell sharply immediately after the price increase.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-007", - "input": "The quarterly revenue exceeded targets but net profit declined by 12%.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-008", - "input": "Revenue from the premium tier dropped while total revenue grew.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-009", - "input": "We need to improve our customer retention rate.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-010", - "input": "The system latency went from 200ms to 5 seconds on Tuesday.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-011", - "input": "The new release should fix the login issue.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-012", - "input": "I think therefore I am.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-013", - "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-014", - "input": "The warehouse manager reported that inventory counts don't match the system.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-015", - "input": "We've seen a 35% increase in customer complaints.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-016", - "input": "The number of active users increased by 500%, from 4 to 2,001.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-017", - "input": "User engagement metrics improved but the support ticket backlog grew by 200%.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-018", - "input": "The manufacturing team needs better quality control.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-019", - "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-020", - "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-021", - "input": "Production output was 1,200 units last month and 1,180 units this month.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-022", - "input": "The CFO reported that the company's cash position is healthy.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-023", - "input": "We have enough funding to operate for 18 months.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-024", - "input": "The new feature was deployed at 3am and user complaints tripled the next day.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-025", - "input": "We need to launch a mobile app to capture market share.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-026", - "input": "The system has been running for 90 days without failure since the migration.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-027", - "input": "No one has submitted the required compliance report despite multiple reminders.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-028", - "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-029", - "input": "We should implement the new CRM because our competitors have one.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-030", - "input": "The server response time was acceptable last quarter but degraded this month.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-031", - "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-032", - "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-033", - "input": "The database schema was changed on Friday but the reports are still working.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-034", - "input": "Some team members say the new process is better while others say it's slower.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - }, - { - "id": "tc-035", - "input": "The application works fine on Chrome but not on Safari.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "identify_difference", - "investigate_contradiction" - ], - "errors": [] - } - ] -} \ No newline at end of file diff --git a/tests-results/evaluation-2026-07-31T18-16-32.json b/tests-results/evaluation-2026-07-31T18-16-32.json deleted file mode 100644 index 4fa8dfd..0000000 --- a/tests-results/evaluation-2026-07-31T18-16-32.json +++ /dev/null @@ -1,652 +0,0 @@ -{ - "timestamp": "2026-07-31T18:16:32.255Z", - "provider": "mock", - "promptVersion": "v0.2", - "casesRun": 35, - "summary": { - "schemaValidityRate": "100.0%", - "classificationMatchRate": "37.1%", - "reasoningModeMatchRate": "71.4%", - "requiredConceptMatchRate": "0.0%", - "unsupportedInferenceFailures": "0", - "averageResponseDurationMs": "1", - "fullPassRate": "37.1%" - }, - "testCaseResults": [ - { - "id": "tc-001", - "input": "All customers cannot download their invoices.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 4, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-002", - "input": "Some customers cannot download their invoices.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-003", - "input": "Complaints increased by 35%.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-004", - "input": "Complaints increased by 35% while production increased by 40%.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-005", - "input": "Sales are falling.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-006", - "input": "Sales fell sharply immediately after the price increase.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-007", - "input": "The quarterly revenue exceeded targets but net profit declined by 12%.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-008", - "input": "Revenue from the premium tier dropped while total revenue grew.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-009", - "input": "We need to improve our customer retention rate.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-010", - "input": "The system latency went from 200ms to 5 seconds on Tuesday.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-011", - "input": "The new release should fix the login issue.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-012", - "input": "I think therefore I am.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "ambiguous_statement", - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "errors": [] - }, - { - "id": "tc-013", - "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "ambiguous_statement", - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "errors": [] - }, - { - "id": "tc-014", - "input": "The warehouse manager reported that inventory counts don't match the system.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-015", - "input": "We've seen a 35% increase in customer complaints.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-016", - "input": "The number of active users increased by 500%, from 4 to 2,001.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-017", - "input": "User engagement metrics improved but the support ticket backlog grew by 200%.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-018", - "input": "The manufacturing team needs better quality control.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-019", - "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-020", - "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-021", - "input": "Production output was 1,200 units last month and 1,180 units this month.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-022", - "input": "The CFO reported that the company's cash position is healthy.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-023", - "input": "We have enough funding to operate for 18 months.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-024", - "input": "The new feature was deployed at 3am and user complaints tripled the next day.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-025", - "input": "We need to launch a mobile app to capture market share.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-026", - "input": "The system has been running for 90 days without failure since the migration.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-027", - "input": "No one has submitted the required compliance report despite multiple reminders.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-028", - "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-029", - "input": "We should implement the new CRM because our competitors have one.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-030", - "input": "The server response time was acceptable last quarter but degraded this month.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-031", - "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-032", - "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-033", - "input": "The database schema was changed on Friday but the reports are still working.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-034", - "input": "Some team members say the new process is better while others say it's slower.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "investigate_contradiction", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-035", - "input": "The application works fine on Chrome but not on Safari.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - } - ] -} \ No newline at end of file diff --git a/tests-results/evaluation-2026-07-31T18-23-11.json b/tests-results/evaluation-2026-07-31T18-23-11.json deleted file mode 100644 index 14d293e..0000000 --- a/tests-results/evaluation-2026-07-31T18-23-11.json +++ /dev/null @@ -1,633 +0,0 @@ -{ - "timestamp": "2026-07-31T18:23:11.669Z", - "provider": "mock", - "promptVersion": "v0.2", - "casesRun": 35, - "summary": { - "schemaValidityRate": "100.0%", - "classificationMatchRate": "48.6%", - "reasoningModeMatchRate": "48.6%", - "requiredConceptMatchRate": "0.0%", - "unsupportedInferenceFailures": "0", - "averageResponseDurationMs": "1", - "fullPassRate": "48.6%" - }, - "testCaseResults": [ - { - "id": "tc-001", - "input": "All customers cannot download their invoices.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 6, - "actualPrimaryType": "fault_report", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-002", - "input": "Some customers cannot download their invoices.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-003", - "input": "Complaints increased by 35%.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "fault_report", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-004", - "input": "Complaints increased by 35% while production increased by 40%.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "fault_report", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-005", - "input": "Sales are falling.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "fault_report", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-006", - "input": "Sales fell sharply immediately after the price increase.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-007", - "input": "The quarterly revenue exceeded targets but net profit declined by 12%.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "other", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-008", - "input": "Revenue from the premium tier dropped while total revenue grew.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-009", - "input": "We need to improve our customer retention rate.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-010", - "input": "The system latency went from 200ms to 5 seconds on Tuesday.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "errors": [] - }, - { - "id": "tc-011", - "input": "The new release should fix the login issue.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "other", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-012", - "input": "I think therefore I am.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "ambiguous_statement", - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "errors": [] - }, - { - "id": "tc-013", - "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "ambiguous_statement", - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "errors": [] - }, - { - "id": "tc-014", - "input": "The warehouse manager reported that inventory counts don't match the system.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-015", - "input": "We've seen a 35% increase in customer complaints.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "fault_report", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-016", - "input": "The number of active users increased by 500%, from 4 to 2,001.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "other", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-017", - "input": "User engagement metrics improved but the support ticket backlog grew by 200%.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "other", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-018", - "input": "The manufacturing team needs better quality control.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "other", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-019", - "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "fault_report", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-020", - "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-021", - "input": "Production output was 1,200 units last month and 1,180 units this month.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "other", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-022", - "input": "The CFO reported that the company's cash position is healthy.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-023", - "input": "We have enough funding to operate for 18 months.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "other", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-024", - "input": "The new feature was deployed at 3am and user complaints tripled the next day.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "fault_report", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-025", - "input": "We need to launch a mobile app to capture market share.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-026", - "input": "The system has been running for 90 days without failure since the migration.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "other", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-027", - "input": "No one has submitted the required compliance report despite multiple reminders.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "fault_report", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-028", - "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "other", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-029", - "input": "We should implement the new CRM because our competitors have one.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-030", - "input": "The server response time was acceptable last quarter but degraded this month.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "errors": [] - }, - { - "id": "tc-031", - "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "fault_report", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-032", - "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "other", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-033", - "input": "The database schema was changed on Friday but the reports are still working.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "other", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-034", - "input": "Some team members say the new process is better while others say it's slower.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "investigate_contradiction", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-035", - "input": "The application works fine on Chrome but not on Safari.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "other", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - } - ] -} \ No newline at end of file diff --git a/tests-results/evaluation-2026-07-31T18-34-17.json b/tests-results/evaluation-2026-07-31T18-34-17.json deleted file mode 100644 index 025baf1..0000000 --- a/tests-results/evaluation-2026-07-31T18-34-17.json +++ /dev/null @@ -1,639 +0,0 @@ -{ - "timestamp": "2026-07-31T18:34:17.855Z", - "provider": "mock", - "promptVersion": "v0.2", - "casesRun": 35, - "summary": { - "schemaValidityRate": "100.0%", - "classificationMatchRate": "82.9%", - "reasoningModeMatchRate": "51.4%", - "requiredConceptMatchRate": "0.0%", - "unsupportedInferenceFailures": "0", - "averageResponseDurationMs": "1", - "fullPassRate": "82.9%" - }, - "testCaseResults": [ - { - "id": "tc-001", - "input": "All customers cannot download their invoices.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 5, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-002", - "input": "Some customers cannot download their invoices.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 2, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-003", - "input": "Complaints increased by 35%.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-004", - "input": "Complaints increased by 35% while production increased by 40%.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-005", - "input": "Sales are falling.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-006", - "input": "Sales fell sharply immediately after the price increase.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-007", - "input": "The quarterly revenue exceeded targets but net profit declined by 12%.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "investigate_contradiction", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-008", - "input": "Revenue from the premium tier dropped while total revenue grew.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "errors": [] - }, - { - "id": "tc-009", - "input": "We need to improve our customer retention rate.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-010", - "input": "The system latency went from 200ms to 5 seconds on Tuesday.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "errors": [] - }, - { - "id": "tc-011", - "input": "The new release should fix the login issue.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-012", - "input": "I think therefore I am.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "ambiguous_statement", - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "errors": [] - }, - { - "id": "tc-013", - "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "ambiguous_statement", - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "errors": [] - }, - { - "id": "tc-014", - "input": "The warehouse manager reported that inventory counts don't match the system.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-015", - "input": "We've seen a 35% increase in customer complaints.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-016", - "input": "The number of active users increased by 500%, from 4 to 2,001.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "errors": [] - }, - { - "id": "tc-017", - "input": "User engagement metrics improved but the support ticket backlog grew by 200%.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "errors": [] - }, - { - "id": "tc-018", - "input": "The manufacturing team needs better quality control.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-019", - "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-020", - "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-021", - "input": "Production output was 1,200 units last month and 1,180 units this month.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-022", - "input": "The CFO reported that the company's cash position is healthy.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-023", - "input": "We have enough funding to operate for 18 months.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-024", - "input": "The new feature was deployed at 3am and user complaints tripled the next day.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-025", - "input": "We need to launch a mobile app to capture market share.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-026", - "input": "The system has been running for 90 days without failure since the migration.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-027", - "input": "No one has submitted the required compliance report despite multiple reminders.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-028", - "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-029", - "input": "We should implement the new CRM because our competitors have one.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-030", - "input": "The server response time was acceptable last quarter but degraded this month.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-031", - "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-032", - "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-033", - "input": "The database schema was changed on Friday but the reports are still working.", - "pass": false, - "schemaValid": true, - "classificationMatch": false, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-034", - "input": "Some team members say the new process is better while others say it's slower.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-035", - "input": "The application works fine on Chrome but not on Safari.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - } - ] -} \ No newline at end of file diff --git a/tests-results/evaluation-2026-07-31T18-59-37.json b/tests-results/evaluation-2026-07-31T18-59-37.json deleted file mode 100644 index 74d39ca..0000000 --- a/tests-results/evaluation-2026-07-31T18-59-37.json +++ /dev/null @@ -1,644 +0,0 @@ -{ - "timestamp": "2026-07-31T18:59:37.227Z", - "provider": "mock", - "promptVersion": "v0.2", - "casesRun": 35, - "summary": { - "schemaValidityRate": "100.0%", - "classificationMatchRate": "100.0%", - "reasoningModeMatchRate": "57.1%", - "requiredConceptMatchRate": "0.0%", - "unsupportedInferenceFailures": "0", - "averageResponseDurationMs": "0", - "fullPassRate": "100.0%" - }, - "testCaseResults": [ - { - "id": "tc-001", - "input": "All customers cannot download their invoices.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 4, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-002", - "input": "Some customers cannot download their invoices.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-003", - "input": "Complaints increased by 35%.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-004", - "input": "Complaints increased by 35% while production increased by 40%.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-005", - "input": "Sales are falling.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-006", - "input": "Sales fell sharply immediately after the price increase.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-007", - "input": "The quarterly revenue exceeded targets but net profit declined by 12%.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "investigate_contradiction", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-008", - "input": "Revenue from the premium tier dropped while total revenue grew.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "errors": [] - }, - { - "id": "tc-009", - "input": "We need to improve our customer retention rate.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-010", - "input": "The system latency went from 200ms to 5 seconds on Tuesday.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "errors": [] - }, - { - "id": "tc-011", - "input": "The new release should fix the login issue.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-012", - "input": "I think therefore I am.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "ambiguous_statement", - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "errors": [] - }, - { - "id": "tc-013", - "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "ambiguous_statement", - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "errors": [] - }, - { - "id": "tc-014", - "input": "The warehouse manager reported that inventory counts don't match the system.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-015", - "input": "We've seen a 35% increase in customer complaints.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-016", - "input": "The number of active users increased by 500%, from 4 to 2,001.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "errors": [] - }, - { - "id": "tc-017", - "input": "User engagement metrics improved but the support ticket backlog grew by 200%.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "investigate_contradiction", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-018", - "input": "The manufacturing team needs better quality control.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-019", - "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-020", - "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-021", - "input": "Production output was 1,200 units last month and 1,180 units this month.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "errors": [] - }, - { - "id": "tc-022", - "input": "The CFO reported that the company's cash position is healthy.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-023", - "input": "We have enough funding to operate for 18 months.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-024", - "input": "The new feature was deployed at 3am and user complaints tripled the next day.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-025", - "input": "We need to launch a mobile app to capture market share.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-026", - "input": "The system has been running for 90 days without failure since the migration.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-027", - "input": "No one has submitted the required compliance report despite multiple reminders.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-028", - "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-029", - "input": "We should implement the new CRM because our competitors have one.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-030", - "input": "The server response time was acceptable last quarter but degraded this month.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-031", - "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-032", - "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-033", - "input": "The database schema was changed on Friday but the reports are still working.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "errors": [] - }, - { - "id": "tc-034", - "input": "Some team members say the new process is better while others say it's slower.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-035", - "input": "The application works fine on Chrome but not on Safari.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - } - ] -} \ No newline at end of file diff --git a/tests-results/evaluation-2026-07-31T19-00-09.json b/tests-results/evaluation-2026-07-31T19-00-09.json deleted file mode 100644 index 7d9ce91..0000000 --- a/tests-results/evaluation-2026-07-31T19-00-09.json +++ /dev/null @@ -1,644 +0,0 @@ -{ - "timestamp": "2026-07-31T19:00:09.575Z", - "provider": "mock", - "promptVersion": "v0.2", - "casesRun": 35, - "summary": { - "schemaValidityRate": "100.0%", - "classificationMatchRate": "100.0%", - "reasoningModeMatchRate": "57.1%", - "requiredConceptMatchRate": "0.0%", - "unsupportedInferenceFailures": "0", - "averageResponseDurationMs": "1", - "fullPassRate": "100.0%" - }, - "testCaseResults": [ - { - "id": "tc-001", - "input": "All customers cannot download their invoices.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 6, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-002", - "input": "Some customers cannot download their invoices.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 2, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-003", - "input": "Complaints increased by 35%.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-004", - "input": "Complaints increased by 35% while production increased by 40%.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-005", - "input": "Sales are falling.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-006", - "input": "Sales fell sharply immediately after the price increase.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-007", - "input": "The quarterly revenue exceeded targets but net profit declined by 12%.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "investigate_contradiction", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-008", - "input": "Revenue from the premium tier dropped while total revenue grew.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "errors": [] - }, - { - "id": "tc-009", - "input": "We need to improve our customer retention rate.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-010", - "input": "The system latency went from 200ms to 5 seconds on Tuesday.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "errors": [] - }, - { - "id": "tc-011", - "input": "The new release should fix the login issue.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-012", - "input": "I think therefore I am.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "ambiguous_statement", - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "errors": [] - }, - { - "id": "tc-013", - "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "ambiguous_statement", - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "errors": [] - }, - { - "id": "tc-014", - "input": "The warehouse manager reported that inventory counts don't match the system.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-015", - "input": "We've seen a 35% increase in customer complaints.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-016", - "input": "The number of active users increased by 500%, from 4 to 2,001.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "errors": [] - }, - { - "id": "tc-017", - "input": "User engagement metrics improved but the support ticket backlog grew by 200%.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "investigate_contradiction", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-018", - "input": "The manufacturing team needs better quality control.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-019", - "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-020", - "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-021", - "input": "Production output was 1,200 units last month and 1,180 units this month.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "errors": [] - }, - { - "id": "tc-022", - "input": "The CFO reported that the company's cash position is healthy.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "errors": [] - }, - { - "id": "tc-023", - "input": "We have enough funding to operate for 18 months.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-024", - "input": "The new feature was deployed at 3am and user complaints tripled the next day.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-025", - "input": "We need to launch a mobile app to capture market share.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-026", - "input": "The system has been running for 90 days without failure since the migration.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-027", - "input": "No one has submitted the required compliance report despite multiple reminders.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-028", - "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-029", - "input": "We should implement the new CRM because our competitors have one.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "errors": [] - }, - { - "id": "tc-030", - "input": "The server response time was acceptable last quarter but degraded this month.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-031", - "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-032", - "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-033", - "input": "The database schema was changed on Friday but the reports are still working.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "errors": [] - }, - { - "id": "tc-034", - "input": "Some team members say the new process is better while others say it's slower.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - }, - { - "id": "tc-035", - "input": "The application works fine on Chrome but not on Safari.", - "pass": true, - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "requiredConceptMatch": false, - "unsupportedInferenceAbsence": true, - "nextQuestionPresent": true, - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "errors": [] - } - ] -} \ No newline at end of file diff --git a/tests-results/evaluation-2026-08-01T05-17-18.json b/tests-results/evaluation-2026-08-01T05-17-18.json deleted file mode 100644 index bda85c4..0000000 --- a/tests-results/evaluation-2026-08-01T05-17-18.json +++ /dev/null @@ -1,1735 +0,0 @@ -{ - "timestamp": "2026-08-01T05:17:18.969Z", - "provider": "mock", - "promptVersion": "v0.2", - "casesRun": 35, - "summary": { - "technical": { - "schemaValidityRate": "100.0%", - "classificationMatchRate": "100.0%", - "nextQuestionPresentRate": "100.0%", - "passRate": "100.0%" - }, - "reasoningQuality": { - "requiredConceptMatchRate": "0.0%", - "unsupportedInferenceFailures": "0", - "passRate": "0.0%" - }, - "combinedPassRate": "0.0%", - "averageResponseDurationMs": "1" - }, - "testCaseResults": [ - { - "id": "tc-001", - "input": "All customers cannot download their invoices.", - "responseDurationMs": 6, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "customers", - "found": false - }, - { - "concept": "invoices", - "found": false - }, - { - "concept": "download", - "found": false - }, - { - "concept": "access_issue", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "tc-002", - "input": "Some customers cannot download their invoices.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "some_customers", - "found": false - }, - { - "concept": "invoices", - "found": false - }, - { - "concept": "download", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "root_cause", - "absent": true - }, - { - "concept": "payment_system_failure", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-003", - "input": "Complaints increased by 35%.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "complaints", - "found": true - }, - { - "concept": "increase", - "found": false - }, - { - "concept": "35_percent", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cause_of_complaints", - "absent": true - }, - { - "concept": "customer_dissatisfaction_is_worse", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-004", - "input": "Complaints increased by 35% while production increased by 40%.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "complaints_increase", - "found": false - }, - { - "concept": "production_increase", - "found": false - }, - { - "concept": "relative_rates", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "production_quality_declined", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-005", - "input": "Sales are falling.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "sales_decline", - "found": false - }, - { - "concept": "direction_negative", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cause_of_fall", - "absent": true - }, - { - "concept": "competitor_action", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-006", - "input": "Sales fell sharply immediately after the price increase.", - "responseDurationMs": 2, - "actualPrimaryType": null, - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "sales_decline", - "found": false - }, - { - "concept": "price_increase", - "found": false - }, - { - "concept": "temporal_correlation", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "price_increase_caused_the_fall", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-007", - "input": "The quarterly revenue exceeded targets but net profit declined by 12%.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "investigate_contradiction", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "revenue_above_target", - "found": false - }, - { - "concept": "profit_decline", - "found": false - }, - { - "concept": "divergence", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cost_overrun_is_the_cause", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-008", - "input": "Revenue from the premium tier dropped while total revenue grew.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "premium_tier_decline", - "found": false - }, - { - "concept": "total_revenue_growth", - "found": false - }, - { - "concept": "segment_cannibalization_risk", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "pricing_change_occurred", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-009", - "input": "We need to improve our customer retention rate.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "retention_improvement_desired", - "found": false - }, - { - "concept": "current_state_unknown", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "retention_rate_is_low", - "absent": true - }, - { - "concept": "churn_has_increased", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-010", - "input": "The system latency went from 200ms to 5 seconds on Tuesday.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "latency_baseline_200ms", - "found": false - }, - { - "concept": "latency_spike_5s", - "found": false - }, - { - "concept": "timestamp_tuesday", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "database_cause", - "absent": true - }, - { - "concept": "release_cause", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-011", - "input": "The new release should fix the login issue.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "proposed_solution", - "found": false - }, - { - "concept": "login_issue", - "found": false - }, - { - "concept": "solution_claim", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "login_issue_is_real", - "absent": true - }, - { - "concept": "release_will_work", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-012", - "input": "I think therefore I am.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "philosophical_statement", - "found": false - }, - { - "concept": "insufficient_operational_context", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "business_problem_exists", - "absent": true - }, - { - "concept": "actionable_insight_possible", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-013", - "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "meta_context", - "found": false - }, - { - "concept": "testing_hypothesis", - "found": false - }, - { - "concept": "self_reference", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "tc-014", - "input": "The warehouse manager reported that inventory counts don't match the system.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "warehouse_manager_report", - "found": false - }, - { - "concept": "inventory_mismatch", - "found": false - }, - { - "concept": "system_discrepancy", - "found": false - }, - { - "concept": "source_attribution", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "theft_occurred", - "absent": true - }, - { - "concept": "software_bug", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-015", - "input": "We've seen a 35% increase in customer complaints.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "complaints_increase", - "found": false - }, - { - "concept": "percentage_metric", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "product_quality_declined", - "absent": true - }, - { - "concept": "customer_satisfaction_drop", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-016", - "input": "The number of active users increased by 500%, from 4 to 2,001.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "active_users_metric", - "found": false - }, - { - "concept": "absolute_vs_relative_growth", - "found": false - }, - { - "concept": "small_base_problem", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "product_success", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-017", - "input": "User engagement metrics improved but the support ticket backlog grew by 200%.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "investigate_contradiction", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "engagement_improvement", - "found": false - }, - { - "concept": "support_backlog_growth", - "found": false - }, - { - "concept": "divergent_metrics", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "users_are_angry", - "absent": true - }, - { - "concept": "product_quality_is_worse", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-018", - "input": "The manufacturing team needs better quality control.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "manufacturing_team", - "found": false - }, - { - "concept": "quality_control_desired", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "quality_is_bad", - "absent": true - }, - { - "concept": "defect_rate_is_high", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-019", - "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "eu_region", - "found": false - }, - { - "concept": "403_error", - "found": false - }, - { - "concept": "access_denied", - "found": false - }, - { - "concept": "geographic_scope", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "gdpr_cause", - "absent": true - }, - { - "concept": "regulatory_change", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-020", - "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "eu_region_subset", - "found": false - }, - { - "concept": "403_error", - "found": false - }, - { - "concept": "partial_reachability", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "all_eu_users_affected", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-021", - "input": "Production output was 1,200 units last month and 1,180 units this month.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "production_output", - "found": false - }, - { - "concept": "month_over_month_decline", - "found": false - }, - { - "concept": "absolute_difference", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "efficiency_loss_occurred", - "absent": true - }, - { - "concept": "equipment_failure", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-022", - "input": "The CFO reported that the company's cash position is healthy.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "cfo_statement", - "found": false - }, - { - "concept": "cash_position_claim", - "found": false - }, - { - "concept": "source_attribution_cfo", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cash_is_healthy", - "absent": true - }, - { - "concept": "financial_stability_is_real", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-023", - "input": "We have enough funding to operate for 18 months.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "funding_period", - "found": false - }, - { - "concept": "operational_sustainability", - "found": false - }, - { - "concept": "burn_rate_unknown", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "no_risk_exists", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-024", - "input": "The new feature was deployed at 3am and user complaints tripled the next day.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "feature_deployment", - "found": false - }, - { - "concept": "timing_3am", - "found": false - }, - { - "concept": "complaint_tripling", - "found": false - }, - { - "concept": "temporal_relationship", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "deployment_caused_complaints", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-025", - "input": "We need to launch a mobile app to capture market share.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "mobile_app_proposed", - "found": false - }, - { - "concept": "market_share_desired", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "no_mobile_app_exists", - "absent": true - }, - { - "concept": "competitors_have_apps", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-026", - "input": "The system has been running for 90 days without failure since the migration.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "uptime_90_days", - "found": false - }, - { - "concept": "post_migration_context", - "found": false - }, - { - "concept": "baseline_established", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "system_is_stable_forever", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-027", - "input": "No one has submitted the required compliance report despite multiple reminders.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "compliance_report", - "found": false - }, - { - "concept": "multiple_reminders", - "found": false - }, - { - "concept": "non_submission", - "found": false - }, - { - "concept": "absent_action", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "deliberate_refusal", - "absent": true - }, - { - "concept": "negligence", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-028", - "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "audit_findings", - "found": false - }, - { - "concept": "incorrect_reports_rate_3_of_10", - "found": false - }, - { - "concept": "data_accuracy_issue", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "intentional_falsification", - "absent": true - }, - { - "concept": "systemic_failure", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-029", - "input": "We should implement the new CRM because our competitors have one.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "crm_proposal", - "found": false - }, - { - "concept": "competitor_comparison", - "found": false - }, - { - "concept": "competitive_pressure", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "crm_will_help", - "absent": true - }, - { - "concept": "we_lack_crm", - "absent": true - }, - { - "concept": "competitors_success_is_from_crm", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-030", - "input": "The server response time was acceptable last quarter but degraded this month.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "response_time_baseline_acceptable", - "found": false - }, - { - "concept": "degradation_timeline", - "found": false - }, - { - "concept": "quarter_to_month_comparison", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "load_increase_occurred", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-031", - "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "regulatory_requirement", - "found": false - }, - { - "concept": "data_location_violation", - "found": false - }, - { - "concept": "cross_border_backup", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "compliance_failure_is_certain", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-032", - "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "industry_decline_prediction", - "found": false - }, - { - "concept": "external_source", - "found": false - }, - { - "concept": "regulatory_cause", - "found": false - }, - { - "concept": "15_percent_forecast", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "decline_will_occur", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-033", - "input": "The database schema was changed on Friday but the reports are still working.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "schema_change", - "found": false - }, - { - "concept": "reports_working_post_change", - "found": false - }, - { - "concept": "unexpected_continuity", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "change_was_harmless", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-034", - "input": "Some team members say the new process is better while others say it's slower.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "subjective_split", - "found": false - }, - { - "concept": "new_process_evaluation", - "found": false - }, - { - "concept": "conflicting_opinions", - "found": false - }, - { - "concept": "measurement_gap", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "process_is_better_or_worse", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-035", - "input": "The application works fine on Chrome but not on Safari.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "chrome_compatibility", - "found": false - }, - { - "concept": "safari_incompatibility", - "found": false - }, - { - "concept": "browser_specific_issue", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "webkit_bug", - "absent": true - } - ] - }, - "pass": false - } - } - ] -} \ No newline at end of file diff --git a/tests-results/evaluation-2026-08-01T05-22-52.json b/tests-results/evaluation-2026-08-01T05-22-52.json deleted file mode 100644 index f5a1685..0000000 --- a/tests-results/evaluation-2026-08-01T05-22-52.json +++ /dev/null @@ -1,1735 +0,0 @@ -{ - "timestamp": "2026-08-01T05:22:52.077Z", - "provider": "mock", - "promptVersion": "v0.2", - "casesRun": 35, - "summary": { - "technical": { - "schemaValidityRate": "100.0%", - "classificationMatchRate": "100.0%", - "nextQuestionPresentRate": "100.0%", - "passRate": "100.0%" - }, - "reasoningQuality": { - "requiredConceptMatchRate": "0.0%", - "unsupportedInferenceFailures": "0", - "passRate": "0.0%" - }, - "combinedPassRate": "0.0%", - "averageResponseDurationMs": "1" - }, - "testCaseResults": [ - { - "id": "tc-001", - "input": "All customers cannot download their invoices.", - "responseDurationMs": 4, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "customers", - "found": false - }, - { - "concept": "invoices", - "found": false - }, - { - "concept": "download", - "found": false - }, - { - "concept": "access_issue", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "tc-002", - "input": "Some customers cannot download their invoices.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "some_customers", - "found": false - }, - { - "concept": "invoices", - "found": false - }, - { - "concept": "download", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "root_cause", - "absent": true - }, - { - "concept": "payment_system_failure", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-003", - "input": "Complaints increased by 35%.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "complaints", - "found": true - }, - { - "concept": "increase", - "found": false - }, - { - "concept": "35_percent", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cause_of_complaints", - "absent": true - }, - { - "concept": "customer_dissatisfaction_is_worse", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-004", - "input": "Complaints increased by 35% while production increased by 40%.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "complaints_increase", - "found": false - }, - { - "concept": "production_increase", - "found": false - }, - { - "concept": "relative_rates", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "production_quality_declined", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-005", - "input": "Sales are falling.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "sales_decline", - "found": false - }, - { - "concept": "direction_negative", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cause_of_fall", - "absent": true - }, - { - "concept": "competitor_action", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-006", - "input": "Sales fell sharply immediately after the price increase.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "sales_decline", - "found": false - }, - { - "concept": "price_increase", - "found": false - }, - { - "concept": "temporal_correlation", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "price_increase_caused_the_fall", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-007", - "input": "The quarterly revenue exceeded targets but net profit declined by 12%.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "investigate_contradiction", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "revenue_above_target", - "found": false - }, - { - "concept": "profit_decline", - "found": false - }, - { - "concept": "divergence", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cost_overrun_is_the_cause", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-008", - "input": "Revenue from the premium tier dropped while total revenue grew.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "premium_tier_decline", - "found": false - }, - { - "concept": "total_revenue_growth", - "found": false - }, - { - "concept": "segment_cannibalization_risk", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "pricing_change_occurred", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-009", - "input": "We need to improve our customer retention rate.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "retention_improvement_desired", - "found": false - }, - { - "concept": "current_state_unknown", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "retention_rate_is_low", - "absent": true - }, - { - "concept": "churn_has_increased", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-010", - "input": "The system latency went from 200ms to 5 seconds on Tuesday.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "latency_baseline_200ms", - "found": false - }, - { - "concept": "latency_spike_5s", - "found": false - }, - { - "concept": "timestamp_tuesday", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "database_cause", - "absent": true - }, - { - "concept": "release_cause", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-011", - "input": "The new release should fix the login issue.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "proposed_solution", - "found": false - }, - { - "concept": "login_issue", - "found": false - }, - { - "concept": "solution_claim", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "login_issue_is_real", - "absent": true - }, - { - "concept": "release_will_work", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-012", - "input": "I think therefore I am.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "philosophical_statement", - "found": false - }, - { - "concept": "insufficient_operational_context", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "business_problem_exists", - "absent": true - }, - { - "concept": "actionable_insight_possible", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-013", - "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "meta_context", - "found": false - }, - { - "concept": "testing_hypothesis", - "found": false - }, - { - "concept": "self_reference", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "tc-014", - "input": "The warehouse manager reported that inventory counts don't match the system.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "warehouse_manager_report", - "found": false - }, - { - "concept": "inventory_mismatch", - "found": false - }, - { - "concept": "system_discrepancy", - "found": false - }, - { - "concept": "source_attribution", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "theft_occurred", - "absent": true - }, - { - "concept": "software_bug", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-015", - "input": "We've seen a 35% increase in customer complaints.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "complaints_increase", - "found": false - }, - { - "concept": "percentage_metric", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "product_quality_declined", - "absent": true - }, - { - "concept": "customer_satisfaction_drop", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-016", - "input": "The number of active users increased by 500%, from 4 to 2,001.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "active_users_metric", - "found": false - }, - { - "concept": "absolute_vs_relative_growth", - "found": false - }, - { - "concept": "small_base_problem", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "product_success", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-017", - "input": "User engagement metrics improved but the support ticket backlog grew by 200%.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "investigate_contradiction", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "engagement_improvement", - "found": false - }, - { - "concept": "support_backlog_growth", - "found": false - }, - { - "concept": "divergent_metrics", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "users_are_angry", - "absent": true - }, - { - "concept": "product_quality_is_worse", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-018", - "input": "The manufacturing team needs better quality control.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "manufacturing_team", - "found": false - }, - { - "concept": "quality_control_desired", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "quality_is_bad", - "absent": true - }, - { - "concept": "defect_rate_is_high", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-019", - "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "eu_region", - "found": false - }, - { - "concept": "403_error", - "found": false - }, - { - "concept": "access_denied", - "found": false - }, - { - "concept": "geographic_scope", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "gdpr_cause", - "absent": true - }, - { - "concept": "regulatory_change", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-020", - "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "eu_region_subset", - "found": false - }, - { - "concept": "403_error", - "found": false - }, - { - "concept": "partial_reachability", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "all_eu_users_affected", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-021", - "input": "Production output was 1,200 units last month and 1,180 units this month.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "production_output", - "found": false - }, - { - "concept": "month_over_month_decline", - "found": false - }, - { - "concept": "absolute_difference", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "efficiency_loss_occurred", - "absent": true - }, - { - "concept": "equipment_failure", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-022", - "input": "The CFO reported that the company's cash position is healthy.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "cfo_statement", - "found": false - }, - { - "concept": "cash_position_claim", - "found": false - }, - { - "concept": "source_attribution_cfo", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cash_is_healthy", - "absent": true - }, - { - "concept": "financial_stability_is_real", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-023", - "input": "We have enough funding to operate for 18 months.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "funding_period", - "found": false - }, - { - "concept": "operational_sustainability", - "found": false - }, - { - "concept": "burn_rate_unknown", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "no_risk_exists", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-024", - "input": "The new feature was deployed at 3am and user complaints tripled the next day.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "feature_deployment", - "found": false - }, - { - "concept": "timing_3am", - "found": false - }, - { - "concept": "complaint_tripling", - "found": false - }, - { - "concept": "temporal_relationship", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "deployment_caused_complaints", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-025", - "input": "We need to launch a mobile app to capture market share.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "mobile_app_proposed", - "found": false - }, - { - "concept": "market_share_desired", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "no_mobile_app_exists", - "absent": true - }, - { - "concept": "competitors_have_apps", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-026", - "input": "The system has been running for 90 days without failure since the migration.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "uptime_90_days", - "found": false - }, - { - "concept": "post_migration_context", - "found": false - }, - { - "concept": "baseline_established", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "system_is_stable_forever", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-027", - "input": "No one has submitted the required compliance report despite multiple reminders.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "compliance_report", - "found": false - }, - { - "concept": "multiple_reminders", - "found": false - }, - { - "concept": "non_submission", - "found": false - }, - { - "concept": "absent_action", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "deliberate_refusal", - "absent": true - }, - { - "concept": "negligence", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-028", - "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "audit_findings", - "found": false - }, - { - "concept": "incorrect_reports_rate_3_of_10", - "found": false - }, - { - "concept": "data_accuracy_issue", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "intentional_falsification", - "absent": true - }, - { - "concept": "systemic_failure", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-029", - "input": "We should implement the new CRM because our competitors have one.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "crm_proposal", - "found": false - }, - { - "concept": "competitor_comparison", - "found": false - }, - { - "concept": "competitive_pressure", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "crm_will_help", - "absent": true - }, - { - "concept": "we_lack_crm", - "absent": true - }, - { - "concept": "competitors_success_is_from_crm", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-030", - "input": "The server response time was acceptable last quarter but degraded this month.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "response_time_baseline_acceptable", - "found": false - }, - { - "concept": "degradation_timeline", - "found": false - }, - { - "concept": "quarter_to_month_comparison", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "load_increase_occurred", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-031", - "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "regulatory_requirement", - "found": false - }, - { - "concept": "data_location_violation", - "found": false - }, - { - "concept": "cross_border_backup", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "compliance_failure_is_certain", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-032", - "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "industry_decline_prediction", - "found": false - }, - { - "concept": "external_source", - "found": false - }, - { - "concept": "regulatory_cause", - "found": false - }, - { - "concept": "15_percent_forecast", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "decline_will_occur", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-033", - "input": "The database schema was changed on Friday but the reports are still working.", - "responseDurationMs": 1, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "schema_change", - "found": false - }, - { - "concept": "reports_working_post_change", - "found": false - }, - { - "concept": "unexpected_continuity", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "change_was_harmless", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-034", - "input": "Some team members say the new process is better while others say it's slower.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "subjective_split", - "found": false - }, - { - "concept": "new_process_evaluation", - "found": false - }, - { - "concept": "conflicting_opinions", - "found": false - }, - { - "concept": "measurement_gap", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "process_is_better_or_worse", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-035", - "input": "The application works fine on Chrome but not on Safari.", - "responseDurationMs": 0, - "actualPrimaryType": null, - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "chrome_compatibility", - "found": false - }, - { - "concept": "safari_incompatibility", - "found": false - }, - { - "concept": "browser_specific_issue", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "webkit_bug", - "absent": true - } - ] - }, - "pass": false - } - } - ] -} \ No newline at end of file diff --git a/tests-results/evaluation-2026-08-01T05-30-52.json b/tests-results/evaluation-2026-08-01T05-30-52.json deleted file mode 100644 index 8d32c0f..0000000 --- a/tests-results/evaluation-2026-08-01T05-30-52.json +++ /dev/null @@ -1,1735 +0,0 @@ -{ - "timestamp": "2026-08-01T05:30:52.558Z", - "provider": "mock", - "promptVersion": "v0.2", - "casesRun": 35, - "summary": { - "technical": { - "schemaValidityRate": "100.0%", - "classificationMatchRate": "100.0%", - "nextQuestionPresentRate": "100.0%", - "passRate": "100.0%" - }, - "reasoningQuality": { - "requiredConceptMatchRate": "0.0%", - "unsupportedInferenceFailures": "0", - "passRate": "0.0%" - }, - "combinedPassRate": "0.0%", - "averageResponseDurationMs": "1" - }, - "testCaseResults": [ - { - "id": "tc-001", - "input": "All customers cannot download their invoices.", - "responseDurationMs": 4, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "customers", - "found": false - }, - { - "concept": "invoices", - "found": false - }, - { - "concept": "download", - "found": false - }, - { - "concept": "access_issue", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "tc-002", - "input": "Some customers cannot download their invoices.", - "responseDurationMs": 2, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "some_customers", - "found": false - }, - { - "concept": "invoices", - "found": false - }, - { - "concept": "download", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "root_cause", - "absent": true - }, - { - "concept": "payment_system_failure", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-003", - "input": "Complaints increased by 35%.", - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "complaints", - "found": true - }, - { - "concept": "increase", - "found": false - }, - { - "concept": "35_percent", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cause_of_complaints", - "absent": true - }, - { - "concept": "customer_dissatisfaction_is_worse", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-004", - "input": "Complaints increased by 35% while production increased by 40%.", - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "complaints_increase", - "found": false - }, - { - "concept": "production_increase", - "found": false - }, - { - "concept": "relative_rates", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "production_quality_declined", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-005", - "input": "Sales are falling.", - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "sales_decline", - "found": false - }, - { - "concept": "direction_negative", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cause_of_fall", - "absent": true - }, - { - "concept": "competitor_action", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-006", - "input": "Sales fell sharply immediately after the price increase.", - "responseDurationMs": 1, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "sales_decline", - "found": false - }, - { - "concept": "price_increase", - "found": false - }, - { - "concept": "temporal_correlation", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "price_increase_caused_the_fall", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-007", - "input": "The quarterly revenue exceeded targets but net profit declined by 12%.", - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "investigate_contradiction", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "revenue_above_target", - "found": false - }, - { - "concept": "profit_decline", - "found": false - }, - { - "concept": "divergence", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cost_overrun_is_the_cause", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-008", - "input": "Revenue from the premium tier dropped while total revenue grew.", - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "premium_tier_decline", - "found": false - }, - { - "concept": "total_revenue_growth", - "found": false - }, - { - "concept": "segment_cannibalization_risk", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "pricing_change_occurred", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-009", - "input": "We need to improve our customer retention rate.", - "responseDurationMs": 1, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "retention_improvement_desired", - "found": false - }, - { - "concept": "current_state_unknown", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "retention_rate_is_low", - "absent": true - }, - { - "concept": "churn_has_increased", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-010", - "input": "The system latency went from 200ms to 5 seconds on Tuesday.", - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "latency_baseline_200ms", - "found": false - }, - { - "concept": "latency_spike_5s", - "found": false - }, - { - "concept": "timestamp_tuesday", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "database_cause", - "absent": true - }, - { - "concept": "release_cause", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-011", - "input": "The new release should fix the login issue.", - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "proposed_solution", - "found": false - }, - { - "concept": "login_issue", - "found": false - }, - { - "concept": "solution_claim", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "login_issue_is_real", - "absent": true - }, - { - "concept": "release_will_work", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-012", - "input": "I think therefore I am.", - "responseDurationMs": 1, - "actualPrimaryType": "ambiguous_statement", - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "philosophical_statement", - "found": false - }, - { - "concept": "insufficient_operational_context", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "business_problem_exists", - "absent": true - }, - { - "concept": "actionable_insight_possible", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-013", - "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.", - "responseDurationMs": 0, - "actualPrimaryType": "ambiguous_statement", - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "meta_context", - "found": false - }, - { - "concept": "testing_hypothesis", - "found": false - }, - { - "concept": "self_reference", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "tc-014", - "input": "The warehouse manager reported that inventory counts don't match the system.", - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "warehouse_manager_report", - "found": false - }, - { - "concept": "inventory_mismatch", - "found": false - }, - { - "concept": "system_discrepancy", - "found": false - }, - { - "concept": "source_attribution", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "theft_occurred", - "absent": true - }, - { - "concept": "software_bug", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-015", - "input": "We've seen a 35% increase in customer complaints.", - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "complaints_increase", - "found": false - }, - { - "concept": "percentage_metric", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "product_quality_declined", - "absent": true - }, - { - "concept": "customer_satisfaction_drop", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-016", - "input": "The number of active users increased by 500%, from 4 to 2,001.", - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "active_users_metric", - "found": false - }, - { - "concept": "absolute_vs_relative_growth", - "found": false - }, - { - "concept": "small_base_problem", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "product_success", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-017", - "input": "User engagement metrics improved but the support ticket backlog grew by 200%.", - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "investigate_contradiction", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "engagement_improvement", - "found": false - }, - { - "concept": "support_backlog_growth", - "found": false - }, - { - "concept": "divergent_metrics", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "users_are_angry", - "absent": true - }, - { - "concept": "product_quality_is_worse", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-018", - "input": "The manufacturing team needs better quality control.", - "responseDurationMs": 1, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "manufacturing_team", - "found": false - }, - { - "concept": "quality_control_desired", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "quality_is_bad", - "absent": true - }, - { - "concept": "defect_rate_is_high", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-019", - "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "eu_region", - "found": false - }, - { - "concept": "403_error", - "found": false - }, - { - "concept": "access_denied", - "found": false - }, - { - "concept": "geographic_scope", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "gdpr_cause", - "absent": true - }, - { - "concept": "regulatory_change", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-020", - "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "eu_region_subset", - "found": false - }, - { - "concept": "403_error", - "found": false - }, - { - "concept": "partial_reachability", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "all_eu_users_affected", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-021", - "input": "Production output was 1,200 units last month and 1,180 units this month.", - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "production_output", - "found": false - }, - { - "concept": "month_over_month_decline", - "found": false - }, - { - "concept": "absolute_difference", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "efficiency_loss_occurred", - "absent": true - }, - { - "concept": "equipment_failure", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-022", - "input": "The CFO reported that the company's cash position is healthy.", - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "cfo_statement", - "found": false - }, - { - "concept": "cash_position_claim", - "found": false - }, - { - "concept": "source_attribution_cfo", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cash_is_healthy", - "absent": true - }, - { - "concept": "financial_stability_is_real", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-023", - "input": "We have enough funding to operate for 18 months.", - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "funding_period", - "found": false - }, - { - "concept": "operational_sustainability", - "found": false - }, - { - "concept": "burn_rate_unknown", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "no_risk_exists", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-024", - "input": "The new feature was deployed at 3am and user complaints tripled the next day.", - "responseDurationMs": 0, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "feature_deployment", - "found": false - }, - { - "concept": "timing_3am", - "found": false - }, - { - "concept": "complaint_tripling", - "found": false - }, - { - "concept": "temporal_relationship", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "deployment_caused_complaints", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-025", - "input": "We need to launch a mobile app to capture market share.", - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "mobile_app_proposed", - "found": false - }, - { - "concept": "market_share_desired", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "no_mobile_app_exists", - "absent": true - }, - { - "concept": "competitors_have_apps", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-026", - "input": "The system has been running for 90 days without failure since the migration.", - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "uptime_90_days", - "found": false - }, - { - "concept": "post_migration_context", - "found": false - }, - { - "concept": "baseline_established", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "system_is_stable_forever", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-027", - "input": "No one has submitted the required compliance report despite multiple reminders.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "compliance_report", - "found": false - }, - { - "concept": "multiple_reminders", - "found": false - }, - { - "concept": "non_submission", - "found": false - }, - { - "concept": "absent_action", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "deliberate_refusal", - "absent": true - }, - { - "concept": "negligence", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-028", - "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "audit_findings", - "found": false - }, - { - "concept": "incorrect_reports_rate_3_of_10", - "found": false - }, - { - "concept": "data_accuracy_issue", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "intentional_falsification", - "absent": true - }, - { - "concept": "systemic_failure", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-029", - "input": "We should implement the new CRM because our competitors have one.", - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "crm_proposal", - "found": false - }, - { - "concept": "competitor_comparison", - "found": false - }, - { - "concept": "competitive_pressure", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "crm_will_help", - "absent": true - }, - { - "concept": "we_lack_crm", - "absent": true - }, - { - "concept": "competitors_success_is_from_crm", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-030", - "input": "The server response time was acceptable last quarter but degraded this month.", - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "response_time_baseline_acceptable", - "found": false - }, - { - "concept": "degradation_timeline", - "found": false - }, - { - "concept": "quarter_to_month_comparison", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "load_increase_occurred", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-031", - "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "regulatory_requirement", - "found": false - }, - { - "concept": "data_location_violation", - "found": false - }, - { - "concept": "cross_border_backup", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "compliance_failure_is_certain", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-032", - "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.", - "responseDurationMs": 0, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "industry_decline_prediction", - "found": false - }, - { - "concept": "external_source", - "found": false - }, - { - "concept": "regulatory_cause", - "found": false - }, - { - "concept": "15_percent_forecast", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "decline_will_occur", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-033", - "input": "The database schema was changed on Friday but the reports are still working.", - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "schema_change", - "found": false - }, - { - "concept": "reports_working_post_change", - "found": false - }, - { - "concept": "unexpected_continuity", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "change_was_harmless", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-034", - "input": "Some team members say the new process is better while others say it's slower.", - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "subjective_split", - "found": false - }, - { - "concept": "new_process_evaluation", - "found": false - }, - { - "concept": "conflicting_opinions", - "found": false - }, - { - "concept": "measurement_gap", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "process_is_better_or_worse", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-035", - "input": "The application works fine on Chrome but not on Safari.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "chrome_compatibility", - "found": false - }, - { - "concept": "safari_incompatibility", - "found": false - }, - { - "concept": "browser_specific_issue", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "webkit_bug", - "absent": true - } - ] - }, - "pass": false - } - } - ] -} \ No newline at end of file diff --git a/tests-results/evaluation-2026-08-01T05-31-49.json b/tests-results/evaluation-2026-08-01T05-31-49.json deleted file mode 100644 index ab319e0..0000000 --- a/tests-results/evaluation-2026-08-01T05-31-49.json +++ /dev/null @@ -1,1735 +0,0 @@ -{ - "timestamp": "2026-08-01T05:31:49.869Z", - "provider": "mock", - "promptVersion": "v0.2", - "casesRun": 35, - "summary": { - "technical": { - "schemaValidityRate": "100.0%", - "classificationMatchRate": "100.0%", - "nextQuestionPresentRate": "100.0%", - "passRate": "100.0%" - }, - "reasoningQuality": { - "requiredConceptMatchRate": "0.0%", - "unsupportedInferenceFailures": "0", - "passRate": "0.0%" - }, - "combinedPassRate": "0.0%", - "averageResponseDurationMs": "1" - }, - "testCaseResults": [ - { - "id": "tc-001", - "input": "All customers cannot download their invoices.", - "responseDurationMs": 5, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "customers", - "found": false - }, - { - "concept": "invoices", - "found": false - }, - { - "concept": "download", - "found": false - }, - { - "concept": "access_issue", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "tc-002", - "input": "Some customers cannot download their invoices.", - "responseDurationMs": 2, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "some_customers", - "found": false - }, - { - "concept": "invoices", - "found": false - }, - { - "concept": "download", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "root_cause", - "absent": true - }, - { - "concept": "payment_system_failure", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-003", - "input": "Complaints increased by 35%.", - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "complaints", - "found": true - }, - { - "concept": "increase", - "found": false - }, - { - "concept": "35_percent", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cause_of_complaints", - "absent": true - }, - { - "concept": "customer_dissatisfaction_is_worse", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-004", - "input": "Complaints increased by 35% while production increased by 40%.", - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "complaints_increase", - "found": false - }, - { - "concept": "production_increase", - "found": false - }, - { - "concept": "relative_rates", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "production_quality_declined", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-005", - "input": "Sales are falling.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "sales_decline", - "found": false - }, - { - "concept": "direction_negative", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cause_of_fall", - "absent": true - }, - { - "concept": "competitor_action", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-006", - "input": "Sales fell sharply immediately after the price increase.", - "responseDurationMs": 1, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "sales_decline", - "found": false - }, - { - "concept": "price_increase", - "found": false - }, - { - "concept": "temporal_correlation", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "price_increase_caused_the_fall", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-007", - "input": "The quarterly revenue exceeded targets but net profit declined by 12%.", - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "investigate_contradiction", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "revenue_above_target", - "found": false - }, - { - "concept": "profit_decline", - "found": false - }, - { - "concept": "divergence", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cost_overrun_is_the_cause", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-008", - "input": "Revenue from the premium tier dropped while total revenue grew.", - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "premium_tier_decline", - "found": false - }, - { - "concept": "total_revenue_growth", - "found": false - }, - { - "concept": "segment_cannibalization_risk", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "pricing_change_occurred", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-009", - "input": "We need to improve our customer retention rate.", - "responseDurationMs": 1, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "retention_improvement_desired", - "found": false - }, - { - "concept": "current_state_unknown", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "retention_rate_is_low", - "absent": true - }, - { - "concept": "churn_has_increased", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-010", - "input": "The system latency went from 200ms to 5 seconds on Tuesday.", - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "latency_baseline_200ms", - "found": false - }, - { - "concept": "latency_spike_5s", - "found": false - }, - { - "concept": "timestamp_tuesday", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "database_cause", - "absent": true - }, - { - "concept": "release_cause", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-011", - "input": "The new release should fix the login issue.", - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "proposed_solution", - "found": false - }, - { - "concept": "login_issue", - "found": false - }, - { - "concept": "solution_claim", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "login_issue_is_real", - "absent": true - }, - { - "concept": "release_will_work", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-012", - "input": "I think therefore I am.", - "responseDurationMs": 1, - "actualPrimaryType": "ambiguous_statement", - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "philosophical_statement", - "found": false - }, - { - "concept": "insufficient_operational_context", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "business_problem_exists", - "absent": true - }, - { - "concept": "actionable_insight_possible", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-013", - "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.", - "responseDurationMs": 0, - "actualPrimaryType": "ambiguous_statement", - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "meta_context", - "found": false - }, - { - "concept": "testing_hypothesis", - "found": false - }, - { - "concept": "self_reference", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "tc-014", - "input": "The warehouse manager reported that inventory counts don't match the system.", - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "warehouse_manager_report", - "found": false - }, - { - "concept": "inventory_mismatch", - "found": false - }, - { - "concept": "system_discrepancy", - "found": false - }, - { - "concept": "source_attribution", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "theft_occurred", - "absent": true - }, - { - "concept": "software_bug", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-015", - "input": "We've seen a 35% increase in customer complaints.", - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "complaints_increase", - "found": false - }, - { - "concept": "percentage_metric", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "product_quality_declined", - "absent": true - }, - { - "concept": "customer_satisfaction_drop", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-016", - "input": "The number of active users increased by 500%, from 4 to 2,001.", - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "active_users_metric", - "found": false - }, - { - "concept": "absolute_vs_relative_growth", - "found": false - }, - { - "concept": "small_base_problem", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "product_success", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-017", - "input": "User engagement metrics improved but the support ticket backlog grew by 200%.", - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "investigate_contradiction", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "engagement_improvement", - "found": false - }, - { - "concept": "support_backlog_growth", - "found": false - }, - { - "concept": "divergent_metrics", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "users_are_angry", - "absent": true - }, - { - "concept": "product_quality_is_worse", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-018", - "input": "The manufacturing team needs better quality control.", - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "manufacturing_team", - "found": false - }, - { - "concept": "quality_control_desired", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "quality_is_bad", - "absent": true - }, - { - "concept": "defect_rate_is_high", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-019", - "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.", - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "eu_region", - "found": false - }, - { - "concept": "403_error", - "found": false - }, - { - "concept": "access_denied", - "found": false - }, - { - "concept": "geographic_scope", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "gdpr_cause", - "absent": true - }, - { - "concept": "regulatory_change", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-020", - "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "eu_region_subset", - "found": false - }, - { - "concept": "403_error", - "found": false - }, - { - "concept": "partial_reachability", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "all_eu_users_affected", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-021", - "input": "Production output was 1,200 units last month and 1,180 units this month.", - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "production_output", - "found": false - }, - { - "concept": "month_over_month_decline", - "found": false - }, - { - "concept": "absolute_difference", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "efficiency_loss_occurred", - "absent": true - }, - { - "concept": "equipment_failure", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-022", - "input": "The CFO reported that the company's cash position is healthy.", - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "cfo_statement", - "found": false - }, - { - "concept": "cash_position_claim", - "found": false - }, - { - "concept": "source_attribution_cfo", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cash_is_healthy", - "absent": true - }, - { - "concept": "financial_stability_is_real", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-023", - "input": "We have enough funding to operate for 18 months.", - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "funding_period", - "found": false - }, - { - "concept": "operational_sustainability", - "found": false - }, - { - "concept": "burn_rate_unknown", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "no_risk_exists", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-024", - "input": "The new feature was deployed at 3am and user complaints tripled the next day.", - "responseDurationMs": 0, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "feature_deployment", - "found": false - }, - { - "concept": "timing_3am", - "found": false - }, - { - "concept": "complaint_tripling", - "found": false - }, - { - "concept": "temporal_relationship", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "deployment_caused_complaints", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-025", - "input": "We need to launch a mobile app to capture market share.", - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "mobile_app_proposed", - "found": false - }, - { - "concept": "market_share_desired", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "no_mobile_app_exists", - "absent": true - }, - { - "concept": "competitors_have_apps", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-026", - "input": "The system has been running for 90 days without failure since the migration.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "uptime_90_days", - "found": false - }, - { - "concept": "post_migration_context", - "found": false - }, - { - "concept": "baseline_established", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "system_is_stable_forever", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-027", - "input": "No one has submitted the required compliance report despite multiple reminders.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "compliance_report", - "found": false - }, - { - "concept": "multiple_reminders", - "found": false - }, - { - "concept": "non_submission", - "found": false - }, - { - "concept": "absent_action", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "deliberate_refusal", - "absent": true - }, - { - "concept": "negligence", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-028", - "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "audit_findings", - "found": false - }, - { - "concept": "incorrect_reports_rate_3_of_10", - "found": false - }, - { - "concept": "data_accuracy_issue", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "intentional_falsification", - "absent": true - }, - { - "concept": "systemic_failure", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-029", - "input": "We should implement the new CRM because our competitors have one.", - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "crm_proposal", - "found": false - }, - { - "concept": "competitor_comparison", - "found": false - }, - { - "concept": "competitive_pressure", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "crm_will_help", - "absent": true - }, - { - "concept": "we_lack_crm", - "absent": true - }, - { - "concept": "competitors_success_is_from_crm", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-030", - "input": "The server response time was acceptable last quarter but degraded this month.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "response_time_baseline_acceptable", - "found": false - }, - { - "concept": "degradation_timeline", - "found": false - }, - { - "concept": "quarter_to_month_comparison", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "load_increase_occurred", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-031", - "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "regulatory_requirement", - "found": false - }, - { - "concept": "data_location_violation", - "found": false - }, - { - "concept": "cross_border_backup", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "compliance_failure_is_certain", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-032", - "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.", - "responseDurationMs": 1, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "industry_decline_prediction", - "found": false - }, - { - "concept": "external_source", - "found": false - }, - { - "concept": "regulatory_cause", - "found": false - }, - { - "concept": "15_percent_forecast", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "decline_will_occur", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-033", - "input": "The database schema was changed on Friday but the reports are still working.", - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "schema_change", - "found": false - }, - { - "concept": "reports_working_post_change", - "found": false - }, - { - "concept": "unexpected_continuity", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "change_was_harmless", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-034", - "input": "Some team members say the new process is better while others say it's slower.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "subjective_split", - "found": false - }, - { - "concept": "new_process_evaluation", - "found": false - }, - { - "concept": "conflicting_opinions", - "found": false - }, - { - "concept": "measurement_gap", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "process_is_better_or_worse", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-035", - "input": "The application works fine on Chrome but not on Safari.", - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "chrome_compatibility", - "found": false - }, - { - "concept": "safari_incompatibility", - "found": false - }, - { - "concept": "browser_specific_issue", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "webkit_bug", - "absent": true - } - ] - }, - "pass": false - } - } - ] -} \ No newline at end of file diff --git a/tests-results/evaluation-2026-08-01T06-02-26.json b/tests-results/evaluation-2026-08-01T06-02-26.json deleted file mode 100644 index 62b6268..0000000 --- a/tests-results/evaluation-2026-08-01T06-02-26.json +++ /dev/null @@ -1,1735 +0,0 @@ -{ - "timestamp": "2026-08-01T06:02:26.238Z", - "provider": "mock", - "promptVersion": "v0.2", - "casesRun": 35, - "summary": { - "technical": { - "schemaValidityRate": "100.0%", - "classificationMatchRate": "100.0%", - "nextQuestionPresentRate": "100.0%", - "passRate": "100.0%" - }, - "reasoningQuality": { - "requiredConceptMatchRate": "0.0%", - "unsupportedInferenceFailures": "0", - "passRate": "0.0%" - }, - "combinedPassRate": "0.0%", - "averageResponseDurationMs": "1" - }, - "testCaseResults": [ - { - "id": "tc-001", - "input": "All customers cannot download their invoices.", - "responseDurationMs": 4, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "customers", - "found": false - }, - { - "concept": "invoices", - "found": false - }, - { - "concept": "download", - "found": false - }, - { - "concept": "access_issue", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "tc-002", - "input": "Some customers cannot download their invoices.", - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "some_customers", - "found": false - }, - { - "concept": "invoices", - "found": false - }, - { - "concept": "download", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "root_cause", - "absent": true - }, - { - "concept": "payment_system_failure", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-003", - "input": "Complaints increased by 35%.", - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "complaints", - "found": true - }, - { - "concept": "increase", - "found": false - }, - { - "concept": "35_percent", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cause_of_complaints", - "absent": true - }, - { - "concept": "customer_dissatisfaction_is_worse", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-004", - "input": "Complaints increased by 35% while production increased by 40%.", - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "complaints_increase", - "found": false - }, - { - "concept": "production_increase", - "found": false - }, - { - "concept": "relative_rates", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "production_quality_declined", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-005", - "input": "Sales are falling.", - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "sales_decline", - "found": false - }, - { - "concept": "direction_negative", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cause_of_fall", - "absent": true - }, - { - "concept": "competitor_action", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-006", - "input": "Sales fell sharply immediately after the price increase.", - "responseDurationMs": 1, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "sales_decline", - "found": false - }, - { - "concept": "price_increase", - "found": false - }, - { - "concept": "temporal_correlation", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "price_increase_caused_the_fall", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-007", - "input": "The quarterly revenue exceeded targets but net profit declined by 12%.", - "responseDurationMs": 1, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "investigate_contradiction", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "revenue_above_target", - "found": false - }, - { - "concept": "profit_decline", - "found": false - }, - { - "concept": "divergence", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cost_overrun_is_the_cause", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-008", - "input": "Revenue from the premium tier dropped while total revenue grew.", - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "premium_tier_decline", - "found": false - }, - { - "concept": "total_revenue_growth", - "found": false - }, - { - "concept": "segment_cannibalization_risk", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "pricing_change_occurred", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-009", - "input": "We need to improve our customer retention rate.", - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "retention_improvement_desired", - "found": false - }, - { - "concept": "current_state_unknown", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "retention_rate_is_low", - "absent": true - }, - { - "concept": "churn_has_increased", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-010", - "input": "The system latency went from 200ms to 5 seconds on Tuesday.", - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "latency_baseline_200ms", - "found": false - }, - { - "concept": "latency_spike_5s", - "found": false - }, - { - "concept": "timestamp_tuesday", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "database_cause", - "absent": true - }, - { - "concept": "release_cause", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-011", - "input": "The new release should fix the login issue.", - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "proposed_solution", - "found": false - }, - { - "concept": "login_issue", - "found": false - }, - { - "concept": "solution_claim", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "login_issue_is_real", - "absent": true - }, - { - "concept": "release_will_work", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-012", - "input": "I think therefore I am.", - "responseDurationMs": 1, - "actualPrimaryType": "ambiguous_statement", - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "philosophical_statement", - "found": false - }, - { - "concept": "insufficient_operational_context", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "business_problem_exists", - "absent": true - }, - { - "concept": "actionable_insight_possible", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-013", - "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.", - "responseDurationMs": 0, - "actualPrimaryType": "ambiguous_statement", - "actualReasoningModes": [ - "identify_difference", - "clarify_meaning" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "meta_context", - "found": false - }, - { - "concept": "testing_hypothesis", - "found": false - }, - { - "concept": "self_reference", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [] - }, - "pass": false - } - }, - { - "id": "tc-014", - "input": "The warehouse manager reported that inventory counts don't match the system.", - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "warehouse_manager_report", - "found": false - }, - { - "concept": "inventory_mismatch", - "found": false - }, - { - "concept": "system_discrepancy", - "found": false - }, - { - "concept": "source_attribution", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "theft_occurred", - "absent": true - }, - { - "concept": "software_bug", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-015", - "input": "We've seen a 35% increase in customer complaints.", - "responseDurationMs": 1, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "complaints_increase", - "found": false - }, - { - "concept": "percentage_metric", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "product_quality_declined", - "absent": true - }, - { - "concept": "customer_satisfaction_drop", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-016", - "input": "The number of active users increased by 500%, from 4 to 2,001.", - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "active_users_metric", - "found": false - }, - { - "concept": "absolute_vs_relative_growth", - "found": false - }, - { - "concept": "small_base_problem", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "product_success", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-017", - "input": "User engagement metrics improved but the support ticket backlog grew by 200%.", - "responseDurationMs": 0, - "actualPrimaryType": "contradiction", - "actualReasoningModes": [ - "investigate_contradiction", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "engagement_improvement", - "found": false - }, - { - "concept": "support_backlog_growth", - "found": false - }, - { - "concept": "divergent_metrics", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "users_are_angry", - "absent": true - }, - { - "concept": "product_quality_is_worse", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-018", - "input": "The manufacturing team needs better quality control.", - "responseDurationMs": 1, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "manufacturing_team", - "found": false - }, - { - "concept": "quality_control_desired", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "quality_is_bad", - "absent": true - }, - { - "concept": "defect_rate_is_high", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-019", - "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "eu_region", - "found": false - }, - { - "concept": "403_error", - "found": false - }, - { - "concept": "access_denied", - "found": false - }, - { - "concept": "geographic_scope", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "gdpr_cause", - "absent": true - }, - { - "concept": "regulatory_change", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-020", - "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "eu_region_subset", - "found": false - }, - { - "concept": "403_error", - "found": false - }, - { - "concept": "partial_reachability", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "all_eu_users_affected", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-021", - "input": "Production output was 1,200 units last month and 1,180 units this month.", - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "production_output", - "found": false - }, - { - "concept": "month_over_month_decline", - "found": false - }, - { - "concept": "absolute_difference", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "efficiency_loss_occurred", - "absent": true - }, - { - "concept": "equipment_failure", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-022", - "input": "The CFO reported that the company's cash position is healthy.", - "responseDurationMs": 0, - "actualPrimaryType": "reported_claim", - "actualReasoningModes": [ - "identify_difference", - "validate_claim" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "cfo_statement", - "found": false - }, - { - "concept": "cash_position_claim", - "found": false - }, - { - "concept": "source_attribution_cfo", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "cash_is_healthy", - "absent": true - }, - { - "concept": "financial_stability_is_real", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-023", - "input": "We have enough funding to operate for 18 months.", - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "funding_period", - "found": false - }, - { - "concept": "operational_sustainability", - "found": false - }, - { - "concept": "burn_rate_unknown", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "no_risk_exists", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-024", - "input": "The new feature was deployed at 3am and user complaints tripled the next day.", - "responseDurationMs": 0, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "establish_baseline", - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "feature_deployment", - "found": false - }, - { - "concept": "timing_3am", - "found": false - }, - { - "concept": "complaint_tripling", - "found": false - }, - { - "concept": "temporal_relationship", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "deployment_caused_complaints", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-025", - "input": "We need to launch a mobile app to capture market share.", - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "mobile_app_proposed", - "found": false - }, - { - "concept": "market_share_desired", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "no_mobile_app_exists", - "absent": true - }, - { - "concept": "competitors_have_apps", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-026", - "input": "The system has been running for 90 days without failure since the migration.", - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "uptime_90_days", - "found": false - }, - { - "concept": "post_migration_context", - "found": false - }, - { - "concept": "baseline_established", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "system_is_stable_forever", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-027", - "input": "No one has submitted the required compliance report despite multiple reminders.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "compliance_report", - "found": false - }, - { - "concept": "multiple_reminders", - "found": false - }, - { - "concept": "non_submission", - "found": false - }, - { - "concept": "absent_action", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "deliberate_refusal", - "absent": true - }, - { - "concept": "negligence", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-028", - "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "audit_findings", - "found": false - }, - { - "concept": "incorrect_reports_rate_3_of_10", - "found": false - }, - { - "concept": "data_accuracy_issue", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "intentional_falsification", - "absent": true - }, - { - "concept": "systemic_failure", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-029", - "input": "We should implement the new CRM because our competitors have one.", - "responseDurationMs": 0, - "actualPrimaryType": "decision_request", - "actualReasoningModes": [ - "identify_difference", - "decision_support", - "identify_missing_information" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "crm_proposal", - "found": false - }, - { - "concept": "competitor_comparison", - "found": false - }, - { - "concept": "competitive_pressure", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "crm_will_help", - "absent": true - }, - { - "concept": "we_lack_crm", - "absent": true - }, - { - "concept": "competitors_success_is_from_crm", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-030", - "input": "The server response time was acceptable last quarter but degraded this month.", - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "response_time_baseline_acceptable", - "found": false - }, - { - "concept": "degradation_timeline", - "found": false - }, - { - "concept": "quarter_to_month_comparison", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "load_increase_occurred", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-031", - "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "regulatory_requirement", - "found": false - }, - { - "concept": "data_location_violation", - "found": false - }, - { - "concept": "cross_border_backup", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "compliance_failure_is_certain", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-032", - "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.", - "responseDurationMs": 0, - "actualPrimaryType": "causal_claim", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "industry_decline_prediction", - "found": false - }, - { - "concept": "external_source", - "found": false - }, - { - "concept": "regulatory_cause", - "found": false - }, - { - "concept": "15_percent_forecast", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "decline_will_occur", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-033", - "input": "The database schema was changed on Friday but the reports are still working.", - "responseDurationMs": 0, - "actualPrimaryType": "unexplained_change", - "actualReasoningModes": [ - "identify_difference", - "establish_baseline", - "validate_measurement" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "schema_change", - "found": false - }, - { - "concept": "reports_working_post_change", - "found": false - }, - { - "concept": "unexpected_continuity", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "change_was_harmless", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-034", - "input": "Some team members say the new process is better while others say it's slower.", - "responseDurationMs": 1, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": false, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "subjective_split", - "found": false - }, - { - "concept": "new_process_evaluation", - "found": false - }, - { - "concept": "conflicting_opinions", - "found": false - }, - { - "concept": "measurement_gap", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "process_is_better_or_worse", - "absent": true - } - ] - }, - "pass": false - } - }, - { - "id": "tc-035", - "input": "The application works fine on Chrome but not on Safari.", - "responseDurationMs": 0, - "actualPrimaryType": "observed_problem", - "actualReasoningModes": [ - "identify_difference" - ], - "technical": { - "schemaValid": true, - "classificationMatch": true, - "reasoningModeMatch": true, - "nextQuestionPresent": true, - "pass": true, - "errors": [] - }, - "reasoningQuality": { - "requiredConcepts": { - "pass": false, - "details": [ - { - "concept": "chrome_compatibility", - "found": false - }, - { - "concept": "safari_incompatibility", - "found": false - }, - { - "concept": "browser_specific_issue", - "found": false - } - ] - }, - "unsupportedInferencesAbsent": { - "pass": true, - "details": [ - { - "concept": "webkit_bug", - "absent": true - } - ] - }, - "pass": false - } - } - ] -} \ No newline at end of file