chore: exclude generated evaluation artifacts from git tracking
Remove three timestamped output directories previously committed in error: - evaluation-results/ (3 run dirs + manifest, 97 files) - provider-debug-results/ (2 debug JSONs) - tests-results/ (7 evaluation outputs) These are regenerative diagnostic logs containing local machine paths and internal IPs — not reusable source, test data, or documentation.
This commit is contained in:
@@ -34,3 +34,8 @@ Thumbs.db
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
# Generated evaluation artifacts (regenerated each run)
|
||||
evaluation-results/
|
||||
provider-debug-results/
|
||||
tests-results/
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
{
|
||||
"id": "diag-01",
|
||||
"description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.",
|
||||
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
||||
"responseDurationMs": 6,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "complaints",
|
||||
"found": true
|
||||
},
|
||||
{
|
||||
"concept": "warehouse",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "baseline comparison",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "quality issue",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "staff turnover",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "training gap",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
# Diagnostic Case: diag-01
|
||||
|
||||
Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
We've seen a spike in complaints from our warehouse team this month compared to last month.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ✅ PASS (1/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: unexplained_change
|
||||
- **Actual Reasoning Modes**: establish_baseline, identify_difference
|
||||
- **Response Duration**: 6ms
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): warehouse, baseline comparison
|
||||
@@ -1,55 +0,0 @@
|
||||
{
|
||||
"id": "diag-02",
|
||||
"description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.",
|
||||
"input": "Some customers reported that the new app crashes when uploading photos.",
|
||||
"responseDurationMs": 2,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "app crashes",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "photo upload",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "some customers",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "all users affected",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "server-side bug",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "Android only",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
# Diagnostic Case: diag-02
|
||||
|
||||
Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Some customers reported that the new app crashes when uploading photos.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ✅ PASS (1/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: observed_problem
|
||||
- **Actual Reasoning Modes**: identify_difference
|
||||
- **Response Duration**: 2ms
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): app crashes, photo upload, some customers
|
||||
@@ -1,60 +0,0 @@
|
||||
{
|
||||
"id": "diag-03",
|
||||
"description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.",
|
||||
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": false,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "sales decline",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "price increase",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "revenue increase",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "CFO report",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "price was set too high",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "competitors gained market share",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "revenue data is wrong",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-03
|
||||
|
||||
Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: causal_claim
|
||||
- **Actual Reasoning Modes**: establish_baseline, identify_difference
|
||||
- **Response Duration**: 1ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
classification mismatch
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): sales decline, price increase, revenue increase, CFO report
|
||||
@@ -1,57 +0,0 @@
|
||||
{
|
||||
"id": "diag-04",
|
||||
"description": "Decision request — forward-looking, needs missing info identification.",
|
||||
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "marketplace app",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "Southeast Asia",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "competitor gap",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "this will definitely succeed",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "we have the resources",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "competitors are struggling",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
# Diagnostic Case: diag-04
|
||||
|
||||
Decision request — forward-looking, needs missing info identification.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ✅ PASS (1/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: decision_request
|
||||
- **Actual Reasoning Modes**: identify_difference, decision_support, identify_missing_information
|
||||
- **Response Duration**: 0ms
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): marketplace app, Southeast Asia, competitor gap
|
||||
@@ -1,57 +0,0 @@
|
||||
{
|
||||
"id": "diag-05",
|
||||
"description": "Unexpected continuity — changed context but no outcome change.",
|
||||
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "supplier change",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "three months ago",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "same defect rate",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "new supplier is worse",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "old supplier was better",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "quality process is broken",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
# Diagnostic Case: diag-05
|
||||
|
||||
Unexpected continuity — changed context but no outcome change.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Our production line changed suppliers three months ago but still delivers the same defect rate as before.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ✅ PASS (1/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: unexplained_change
|
||||
- **Actual Reasoning Modes**: identify_difference, establish_baseline, validate_measurement
|
||||
- **Response Duration**: 1ms
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): supplier change, three months ago, same defect rate
|
||||
@@ -1,59 +0,0 @@
|
||||
{
|
||||
"id": "diag-06",
|
||||
"description": "Quantified improvement — needs context about measurement period and baseline conditions.",
|
||||
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": false,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "completion rate",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "45%",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "62%",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "onboarding",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "all improvements are due to the redesign",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "the old flow was bad",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "users prefer the new design",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-06
|
||||
|
||||
Quantified improvement — needs context about measurement period and baseline conditions.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
From 45% to 62%, the completion rate for our onboarding flow improved significantly.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: observed_problem
|
||||
- **Actual Reasoning Modes**: identify_difference
|
||||
- **Response Duration**: 1ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
classification mismatch
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): completion rate, 45%, 62%, onboarding
|
||||
@@ -1,56 +0,0 @@
|
||||
{
|
||||
"id": "diag-07",
|
||||
"description": "Single reported claim — needs validation, not acceptance as fact.",
|
||||
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "pricing complexity",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "small business",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "user claim",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "the pricing is actually complex",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "other small businesses agree",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "we should simplify pricing",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
# Diagnostic Case: diag-07
|
||||
|
||||
Single reported claim — needs validation, not acceptance as fact.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
A user claimed that our pricing model is too complex for small businesses.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ✅ PASS (1/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: reported_claim
|
||||
- **Actual Reasoning Modes**: identify_difference, validate_claim
|
||||
- **Response Duration**: 1ms
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): pricing complexity, small business, user claim
|
||||
@@ -1,56 +0,0 @@
|
||||
{
|
||||
"id": "diag-08",
|
||||
"description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode.",
|
||||
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "philosophical",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "ambiguous",
|
||||
"found": true
|
||||
},
|
||||
{
|
||||
"concept": "meaning clarification",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "the phrase was wrong",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "the colleague is hostile",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "we should avoid philosophical language",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
# Diagnostic Case: diag-08
|
||||
|
||||
Meta-test — self-referential ambiguous statement. Should trigger clarification mode.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ✅ PASS (1/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: ambiguous_statement
|
||||
- **Actual Reasoning Modes**: identify_difference, clarify_meaning
|
||||
- **Response Duration**: 1ms
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): philosophical, meaning clarification
|
||||
@@ -1,60 +0,0 @@
|
||||
{
|
||||
"id": "diag-09",
|
||||
"description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.",
|
||||
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "deployment",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "complaint volume increase",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "tripled",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "47 cases",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "the deployment caused the complaints",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "the bug report was insufficient",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "rollback is needed",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
# Diagnostic Case: diag-09
|
||||
|
||||
Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
After the deployment last week, our complaint volume tripled to 47 cases per day.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ✅ PASS (1/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: causal_claim
|
||||
- **Actual Reasoning Modes**: establish_baseline, identify_difference
|
||||
- **Response Duration**: 1ms
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): deployment, complaint volume increase, tripled, 47 cases
|
||||
@@ -1,56 +0,0 @@
|
||||
{
|
||||
"id": "diag-10",
|
||||
"description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims.",
|
||||
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "production issues",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "delivery speed",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "complaint types",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "production is worse than delivery",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "the delivery team needs training",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "both teams are underperforming equally",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
# Diagnostic Case: diag-10
|
||||
|
||||
Paired with diag-01 — distinguishes subset complaints from aggregate claims.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Some complaints involve production issues, but others say the delivery team is slow.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ✅ PASS (1/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: observed_problem
|
||||
- **Actual Reasoning Modes**: establish_baseline, identify_difference
|
||||
- **Response Duration**: 1ms
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): production issues, delivery speed, complaint types
|
||||
@@ -1,585 +0,0 @@
|
||||
{
|
||||
"timestamp": "2026-08-01T06:01:56.070Z",
|
||||
"provider": "mock",
|
||||
"promptVersion": "v0.2",
|
||||
"casesRun": 10,
|
||||
"summary": {
|
||||
"technical": {
|
||||
"schemaValidityRate": "100.0%",
|
||||
"classificationMatchRate": "80.0%",
|
||||
"nextQuestionPresentRate": "100.0%",
|
||||
"passRate": "80.0%"
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConceptMatchRate": "0.0%",
|
||||
"unsupportedInferenceFailures": "0",
|
||||
"passRate": "0.0%"
|
||||
},
|
||||
"combinedPassRate": "0.0%",
|
||||
"averageResponseDurationMs": "2"
|
||||
},
|
||||
"testCaseResults": [
|
||||
{
|
||||
"id": "diag-01",
|
||||
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
||||
"responseDurationMs": 6,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "complaints",
|
||||
"found": true
|
||||
},
|
||||
{
|
||||
"concept": "warehouse",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "baseline comparison",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "quality issue",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "staff turnover",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "training gap",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-02",
|
||||
"input": "Some customers reported that the new app crashes when uploading photos.",
|
||||
"responseDurationMs": 2,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "app crashes",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "photo upload",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "some customers",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "all users affected",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "server-side bug",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "Android only",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-03",
|
||||
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": false,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "sales decline",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "price increase",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "revenue increase",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "CFO report",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "price was set too high",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "competitors gained market share",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "revenue data is wrong",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-04",
|
||||
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "marketplace app",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "Southeast Asia",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "competitor gap",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "this will definitely succeed",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "we have the resources",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "competitors are struggling",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-05",
|
||||
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "supplier change",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "three months ago",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "same defect rate",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "new supplier is worse",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "old supplier was better",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "quality process is broken",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-06",
|
||||
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": false,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "completion rate",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "45%",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "62%",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "onboarding",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "all improvements are due to the redesign",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "the old flow was bad",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "users prefer the new design",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-07",
|
||||
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "pricing complexity",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "small business",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "user claim",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "the pricing is actually complex",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "other small businesses agree",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "we should simplify pricing",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-08",
|
||||
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "philosophical",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "ambiguous",
|
||||
"found": true
|
||||
},
|
||||
{
|
||||
"concept": "meaning clarification",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "the phrase was wrong",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "the colleague is hostile",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "we should avoid philosophical language",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-09",
|
||||
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "deployment",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "complaint volume increase",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "tripled",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "47 cases",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "the deployment caused the complaints",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "the bug report was insufficient",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "rollback is needed",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-10",
|
||||
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "production issues",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "delivery speed",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "complaint types",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "production is worse than delivery",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "the delivery team needs training",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "both teams are underperforming equally",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
{
|
||||
"id": "diag-01",
|
||||
"description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.",
|
||||
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
||||
"responseDurationMs": 1785564462515,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-01
|
||||
|
||||
Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
We've seen a spike in complaints from our warehouse team this month compared to last month.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462515ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
{
|
||||
"id": "diag-02",
|
||||
"description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.",
|
||||
"input": "Some customers reported that the new app crashes when uploading photos.",
|
||||
"responseDurationMs": 1785564462519,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-02
|
||||
|
||||
Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Some customers reported that the new app crashes when uploading photos.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462519ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
{
|
||||
"id": "diag-03",
|
||||
"description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.",
|
||||
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
||||
"responseDurationMs": 1785564462519,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-03
|
||||
|
||||
Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462519ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
{
|
||||
"id": "diag-04",
|
||||
"description": "Decision request — forward-looking, needs missing info identification.",
|
||||
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
||||
"responseDurationMs": 1785564462520,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-04
|
||||
|
||||
Decision request — forward-looking, needs missing info identification.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462520ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
{
|
||||
"id": "diag-05",
|
||||
"description": "Unexpected continuity — changed context but no outcome change.",
|
||||
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
||||
"responseDurationMs": 1785564462520,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-05
|
||||
|
||||
Unexpected continuity — changed context but no outcome change.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Our production line changed suppliers three months ago but still delivers the same defect rate as before.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462520ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
{
|
||||
"id": "diag-06",
|
||||
"description": "Quantified improvement — needs context about measurement period and baseline conditions.",
|
||||
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
||||
"responseDurationMs": 1785564462521,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-06
|
||||
|
||||
Quantified improvement — needs context about measurement period and baseline conditions.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
From 45% to 62%, the completion rate for our onboarding flow improved significantly.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462521ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
{
|
||||
"id": "diag-07",
|
||||
"description": "Single reported claim — needs validation, not acceptance as fact.",
|
||||
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
||||
"responseDurationMs": 1785564462521,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-07
|
||||
|
||||
Single reported claim — needs validation, not acceptance as fact.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
A user claimed that our pricing model is too complex for small businesses.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462521ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
{
|
||||
"id": "diag-08",
|
||||
"description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode.",
|
||||
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
||||
"responseDurationMs": 1785564462521,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-08
|
||||
|
||||
Meta-test — self-referential ambiguous statement. Should trigger clarification mode.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462521ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
{
|
||||
"id": "diag-09",
|
||||
"description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.",
|
||||
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
||||
"responseDurationMs": 1785564462521,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-09
|
||||
|
||||
Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
After the deployment last week, our complaint volume tripled to 47 cases per day.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462521ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
{
|
||||
"id": "diag-10",
|
||||
"description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims.",
|
||||
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
||||
"responseDurationMs": 1785564462522,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-10
|
||||
|
||||
Paired with diag-01 — distinguishes subset complaints from aggregate claims.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Some complaints involve production issues, but others say the delivery team is slow.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462522ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,313 +0,0 @@
|
||||
{
|
||||
"timestamp": "2026-08-01T06:07:42.532Z",
|
||||
"provider": "ollama-real",
|
||||
"promptVersion": "v0.2",
|
||||
"casesRun": 10,
|
||||
"summary": {
|
||||
"technical": {
|
||||
"schemaValidityRate": "0.0%",
|
||||
"classificationMatchRate": "0.0%",
|
||||
"nextQuestionPresentRate": "0.0%",
|
||||
"passRate": "0.0%"
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConceptMatchRate": "100.0%",
|
||||
"unsupportedInferenceFailures": "0",
|
||||
"passRate": "0.0%"
|
||||
},
|
||||
"combinedPassRate": "0.0%",
|
||||
"averageResponseDurationMs": "1785564462520"
|
||||
},
|
||||
"testCaseResults": [
|
||||
{
|
||||
"id": "diag-01",
|
||||
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
||||
"responseDurationMs": 1785564462515,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-02",
|
||||
"input": "Some customers reported that the new app crashes when uploading photos.",
|
||||
"responseDurationMs": 1785564462519,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-03",
|
||||
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
||||
"responseDurationMs": 1785564462519,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-04",
|
||||
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
||||
"responseDurationMs": 1785564462520,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-05",
|
||||
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
||||
"responseDurationMs": 1785564462520,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-06",
|
||||
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
||||
"responseDurationMs": 1785564462521,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-07",
|
||||
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
||||
"responseDurationMs": 1785564462521,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-08",
|
||||
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
||||
"responseDurationMs": 1785564462521,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-09",
|
||||
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
||||
"responseDurationMs": 1785564462521,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-10",
|
||||
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
||||
"responseDurationMs": 1785564462522,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"id": "diag-01",
|
||||
"description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.",
|
||||
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
||||
"responseDurationMs": 19459,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"rawOutput": "{\"status\":\"received\",\"message\":\"Please provide a specific request or data to process.\"}",
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-01
|
||||
|
||||
Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
We've seen a spike in complaints from our warehouse team this month compared to last month.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 19459ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
{
|
||||
"id": "diag-02",
|
||||
"description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.",
|
||||
"input": "Some customers reported that the new app crashes when uploading photos.",
|
||||
"responseDurationMs": 14693,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download",
|
||||
"Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-02
|
||||
|
||||
Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Some customers reported that the new app crashes when uploading photos.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 14693ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"id": "diag-03",
|
||||
"description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.",
|
||||
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
||||
"responseDurationMs": 13516,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"rawOutput": "{\"status\":\"success\",\"message\":\"Input received and processed.\"}",
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-03
|
||||
|
||||
Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 13516ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"id": "diag-04",
|
||||
"description": "Decision request — forward-looking, needs missing info identification.",
|
||||
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
||||
"responseDurationMs": 48851,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"rawOutput": "{\"state\":\"pending\"}",
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-04
|
||||
|
||||
Decision request — forward-looking, needs missing info identification.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 48851ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"id": "diag-05",
|
||||
"description": "Unexpected continuity — changed context but no outcome change.",
|
||||
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
||||
"responseDurationMs": 16508,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"rawOutput": "{}",
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-05
|
||||
|
||||
Unexpected continuity — changed context but no outcome change.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Our production line changed suppliers three months ago but still delivers the same defect rate as before.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 16508ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"id": "diag-06",
|
||||
"description": "Quantified improvement — needs context about measurement period and baseline conditions.",
|
||||
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
||||
"responseDurationMs": 15608,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"rawOutput": "{\"status\":\"ok\"}",
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-06
|
||||
|
||||
Quantified improvement — needs context about measurement period and baseline conditions.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
From 45% to 62%, the completion rate for our onboarding flow improved significantly.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 15608ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"id": "diag-07",
|
||||
"description": "Single reported claim — needs validation, not acceptance as fact.",
|
||||
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
||||
"responseDurationMs": 15750,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"rawOutput": "{\"status\":\"success\",\"message\":\"Input received\"}",
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-07
|
||||
|
||||
Single reported claim — needs validation, not acceptance as fact.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
A user claimed that our pricing model is too complex for small businesses.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 15750ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"id": "diag-08",
|
||||
"description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode.",
|
||||
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
||||
"responseDurationMs": 15383,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"rawOutput": "{\"status\":\"success\",\"message\":\"JSON object returned as requested\"}",
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-08
|
||||
|
||||
Meta-test — self-referential ambiguous statement. Should trigger clarification mode.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 15383ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"id": "diag-09",
|
||||
"description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.",
|
||||
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
||||
"responseDurationMs": 15643,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"rawOutput": "{\"status\":\"ok\"}",
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-09
|
||||
|
||||
Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
After the deployment last week, our complaint volume tripled to 47 cases per day.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 15643ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
{
|
||||
"id": "diag-10",
|
||||
"description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims.",
|
||||
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
||||
"responseDurationMs": 15401,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"rawOutput": "{}",
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
# Diagnostic Case: diag-10
|
||||
|
||||
Paired with diag-01 — distinguishes subset complaints from aggregate claims.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Some complaints involve production issues, but others say the delivery team is slow.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 15401ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -1,331 +0,0 @@
|
||||
{
|
||||
"timestamp": "2026-08-01T06:11:41.538Z",
|
||||
"provider": "ollama-real",
|
||||
"promptVersion": "v0.2",
|
||||
"casesRun": 10,
|
||||
"summary": {
|
||||
"technical": {
|
||||
"schemaValidityRate": "0.0%",
|
||||
"classificationMatchRate": "0.0%",
|
||||
"nextQuestionPresentRate": "0.0%",
|
||||
"passRate": "0.0%"
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConceptMatchRate": "100.0%",
|
||||
"unsupportedInferenceFailures": "0",
|
||||
"passRate": "0.0%"
|
||||
},
|
||||
"combinedPassRate": "0.0%",
|
||||
"averageResponseDurationMs": "19081"
|
||||
},
|
||||
"testCaseResults": [
|
||||
{
|
||||
"id": "diag-01",
|
||||
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
||||
"responseDurationMs": 19459,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-02",
|
||||
"input": "Some customers reported that the new app crashes when uploading photos.",
|
||||
"responseDurationMs": 14693,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download",
|
||||
"Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-03",
|
||||
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
||||
"responseDurationMs": 13516,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-04",
|
||||
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
||||
"responseDurationMs": 48851,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-05",
|
||||
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
||||
"responseDurationMs": 16508,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-06",
|
||||
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
||||
"responseDurationMs": 15608,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-07",
|
||||
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
||||
"responseDurationMs": 15750,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-08",
|
||||
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
||||
"responseDurationMs": 15383,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-09",
|
||||
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
||||
"responseDurationMs": 15643,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-10",
|
||||
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
||||
"responseDurationMs": 15401,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
{
|
||||
"latestRun": "2026-08-01T06-11-41",
|
||||
"caseCount": 10
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
{
|
||||
"qwen-claude:latest": {
|
||||
"availability": "NOT_AVAILABLE_ON_SERVER"
|
||||
},
|
||||
"qwen3.6:35b-a3b": {
|
||||
"availability": "NOT_AVAILABLE_ON_SERVER"
|
||||
}
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
{
|
||||
"qwen-claude:latest": {
|
||||
"test_A_qwen_claude": {
|
||||
"testDescription": "Plain instruction test — should return CHAT_WORKS",
|
||||
"endpoint": "/api/chat",
|
||||
"format": "json",
|
||||
"hasSystemMessage": true,
|
||||
"httpStatus": 200,
|
||||
"messageContentType": "string",
|
||||
"messageContentLength": 30,
|
||||
"thinkingPresent": true,
|
||||
"parsedContentKeys": [
|
||||
"response"
|
||||
],
|
||||
"looksLikeStatusAck": false,
|
||||
"rawPreview": "{\n \"response\": \"CHAT_WORKS\"\n}"
|
||||
},
|
||||
"test_A_qwen_claude_generate": {
|
||||
"endpoint": "/api/generate",
|
||||
"httpStatus": 200,
|
||||
"responseFirst200": "CHAT_WORKS",
|
||||
"responseLooksLikeStructuredJSON": false,
|
||||
"rawPreview": "CHAT_WORKS"
|
||||
},
|
||||
"test_B_qwen_claude": {
|
||||
"testDescription": "JSON schema test — should return exact object",
|
||||
"endpoint": "/api/chat",
|
||||
"format": "json",
|
||||
"hasSystemMessage": false,
|
||||
"httpStatus": 200,
|
||||
"messageContentType": "string",
|
||||
"messageContentLength": 38,
|
||||
"thinkingPresent": true,
|
||||
"parsedContentKeys": [
|
||||
"message"
|
||||
],
|
||||
"looksLikeStatusAck": false,
|
||||
"rawPreview": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}"
|
||||
},
|
||||
"test_B_qwen_claude_generate": {
|
||||
"endpoint": "/api/generate",
|
||||
"httpStatus": 200,
|
||||
"responseFirst200": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}",
|
||||
"responseLooksLikeStructuredJSON": true,
|
||||
"rawPreview": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}"
|
||||
},
|
||||
"test_C_qwen_claude": {
|
||||
"testDescription": "Minimal reconstruction schema — structured output test",
|
||||
"endpoint": "/api/chat",
|
||||
"format": "json",
|
||||
"hasSystemMessage": false,
|
||||
"httpStatus": 200,
|
||||
"messageContentType": "string",
|
||||
"messageContentLength": 1145,
|
||||
"thinkingPresent": true,
|
||||
"parsedContentKeys": [
|
||||
"analysis",
|
||||
"meaningful_difference",
|
||||
"next_question"
|
||||
],
|
||||
"looksLikeStatusAck": false,
|
||||
"rawPreview": "{\"analysis\":\"The workflow breaks into two distinct stages: authentication (login) and resource access (invoice download). Since login succeeds for the affected users, identity verification and session creation are functioning correctly. The failure occurs downstream in the layer responsible for loca"
|
||||
},
|
||||
"test_C_qwen_claude_generate": {
|
||||
"endpoint": "/api/generate",
|
||||
"httpStatus": 200,
|
||||
"responseFirst200": "**Analysis of the Situation**\nThe issue represents a partial, post-authentication failure: identity verification works for a subset of users, but access to or delivery of a specific resource (invoices",
|
||||
"responseLooksLikeStructuredJSON": false,
|
||||
"rawPreview": "**Analysis of the Situation**\nThe issue represents a partial, post-authentication failure: identity verification works for a subset of users, but access to or delivery of a specific resource (invoices"
|
||||
}
|
||||
},
|
||||
"qwen3.6:35b-a3b": {
|
||||
"test_A_qwen___": {
|
||||
"testDescription": "Plain instruction test — should return CHAT_WORKS",
|
||||
"endpoint": "/api/chat",
|
||||
"format": "json",
|
||||
"hasSystemMessage": true,
|
||||
"httpStatus": 200,
|
||||
"messageContentType": "string",
|
||||
"messageContentLength": 30,
|
||||
"thinkingPresent": true,
|
||||
"parsedContentKeys": [
|
||||
"response"
|
||||
],
|
||||
"looksLikeStatusAck": false,
|
||||
"rawPreview": "{\n \"response\": \"CHAT_WORKS\"\n}"
|
||||
},
|
||||
"test_A_qwen____generate": {
|
||||
"endpoint": "/api/generate",
|
||||
"httpStatus": 200,
|
||||
"responseFirst200": "CHAT_WORKS",
|
||||
"responseLooksLikeStructuredJSON": false,
|
||||
"rawPreview": "CHAT_WORKS"
|
||||
},
|
||||
"test_B_qwen___": {
|
||||
"testDescription": "JSON schema test — should return exact object",
|
||||
"endpoint": "/api/chat",
|
||||
"format": "json",
|
||||
"hasSystemMessage": false,
|
||||
"httpStatus": 200,
|
||||
"messageContentType": "string",
|
||||
"messageContentLength": 38,
|
||||
"thinkingPresent": true,
|
||||
"parsedContentKeys": [
|
||||
"message"
|
||||
],
|
||||
"looksLikeStatusAck": false,
|
||||
"rawPreview": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}"
|
||||
},
|
||||
"test_B_qwen____generate": {
|
||||
"endpoint": "/api/generate",
|
||||
"httpStatus": 200,
|
||||
"responseFirst200": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}",
|
||||
"responseLooksLikeStructuredJSON": true,
|
||||
"rawPreview": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}"
|
||||
},
|
||||
"test_C_qwen___": {
|
||||
"testDescription": "Minimal reconstruction schema — structured output test",
|
||||
"endpoint": "/api/chat",
|
||||
"format": "json",
|
||||
"hasSystemMessage": false,
|
||||
"httpStatus": 200,
|
||||
"messageContentType": "string",
|
||||
"messageContentLength": 843,
|
||||
"thinkingPresent": true,
|
||||
"parsedContentKeys": [
|
||||
"analysis",
|
||||
"meaningful_difference",
|
||||
"next_question"
|
||||
],
|
||||
"looksLikeStatusAck": false,
|
||||
"rawPreview": "{\"analysis\": \"The scenario indicates that authentication is functioning correctly, but a downstream access or resource-retrieval step is failing for a subset of users. This separates the problem from credential or session validity and points toward post-login factors such as authorization scope, doc"
|
||||
},
|
||||
"test_C_qwen____generate": {
|
||||
"endpoint": "/api/generate",
|
||||
"httpStatus": 200,
|
||||
"responseFirst200": "**Meaningful Difference:** \nSuccessful login verifies *account-level authentication and baseline platform access*, while downloading an invoice requires *invoice-specific data availability, retrieval",
|
||||
"responseLooksLikeStructuredJSON": false,
|
||||
"rawPreview": "**Meaningful Difference:** \nSuccessful login verifies *account-level authentication and baseline platform access*, while downloading an invoice requires *invoice-specific data availability, retrieval"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,653 +0,0 @@
|
||||
{
|
||||
"timestamp": "2026-07-31T18:14:22.632Z",
|
||||
"provider": "mock",
|
||||
"promptVersion": "v0.2",
|
||||
"casesRun": 35,
|
||||
"summary": {
|
||||
"schemaValidityRate": "100.0%",
|
||||
"classificationMatchRate": "28.6%",
|
||||
"reasoningModeMatchRate": "48.6%",
|
||||
"requiredConceptMatchRate": "0.0%",
|
||||
"unsupportedInferenceFailures": "0",
|
||||
"averageResponseDurationMs": "1",
|
||||
"fullPassRate": "28.6%"
|
||||
},
|
||||
"testCaseResults": [
|
||||
{
|
||||
"id": "tc-001",
|
||||
"input": "All customers cannot download their invoices.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 4,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-002",
|
||||
"input": "Some customers cannot download their invoices.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-003",
|
||||
"input": "Complaints increased by 35%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-004",
|
||||
"input": "Complaints increased by 35% while production increased by 40%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-005",
|
||||
"input": "Sales are falling.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-006",
|
||||
"input": "Sales fell sharply immediately after the price increase.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-007",
|
||||
"input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-008",
|
||||
"input": "Revenue from the premium tier dropped while total revenue grew.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-009",
|
||||
"input": "We need to improve our customer retention rate.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-010",
|
||||
"input": "The system latency went from 200ms to 5 seconds on Tuesday.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-011",
|
||||
"input": "The new release should fix the login issue.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-012",
|
||||
"input": "I think therefore I am.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-013",
|
||||
"input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-014",
|
||||
"input": "The warehouse manager reported that inventory counts don't match the system.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-015",
|
||||
"input": "We've seen a 35% increase in customer complaints.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-016",
|
||||
"input": "The number of active users increased by 500%, from 4 to 2,001.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-017",
|
||||
"input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-018",
|
||||
"input": "The manufacturing team needs better quality control.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-019",
|
||||
"input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-020",
|
||||
"input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-021",
|
||||
"input": "Production output was 1,200 units last month and 1,180 units this month.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-022",
|
||||
"input": "The CFO reported that the company's cash position is healthy.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-023",
|
||||
"input": "We have enough funding to operate for 18 months.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-024",
|
||||
"input": "The new feature was deployed at 3am and user complaints tripled the next day.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-025",
|
||||
"input": "We need to launch a mobile app to capture market share.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-026",
|
||||
"input": "The system has been running for 90 days without failure since the migration.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-027",
|
||||
"input": "No one has submitted the required compliance report despite multiple reminders.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-028",
|
||||
"input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-029",
|
||||
"input": "We should implement the new CRM because our competitors have one.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-030",
|
||||
"input": "The server response time was acceptable last quarter but degraded this month.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-031",
|
||||
"input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-032",
|
||||
"input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-033",
|
||||
"input": "The database schema was changed on Friday but the reports are still working.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-034",
|
||||
"input": "Some team members say the new process is better while others say it's slower.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-035",
|
||||
"input": "The application works fine on Chrome but not on Safari.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,652 +0,0 @@
|
||||
{
|
||||
"timestamp": "2026-07-31T18:16:32.255Z",
|
||||
"provider": "mock",
|
||||
"promptVersion": "v0.2",
|
||||
"casesRun": 35,
|
||||
"summary": {
|
||||
"schemaValidityRate": "100.0%",
|
||||
"classificationMatchRate": "37.1%",
|
||||
"reasoningModeMatchRate": "71.4%",
|
||||
"requiredConceptMatchRate": "0.0%",
|
||||
"unsupportedInferenceFailures": "0",
|
||||
"averageResponseDurationMs": "1",
|
||||
"fullPassRate": "37.1%"
|
||||
},
|
||||
"testCaseResults": [
|
||||
{
|
||||
"id": "tc-001",
|
||||
"input": "All customers cannot download their invoices.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 4,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-002",
|
||||
"input": "Some customers cannot download their invoices.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-003",
|
||||
"input": "Complaints increased by 35%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-004",
|
||||
"input": "Complaints increased by 35% while production increased by 40%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-005",
|
||||
"input": "Sales are falling.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-006",
|
||||
"input": "Sales fell sharply immediately after the price increase.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-007",
|
||||
"input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-008",
|
||||
"input": "Revenue from the premium tier dropped while total revenue grew.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-009",
|
||||
"input": "We need to improve our customer retention rate.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-010",
|
||||
"input": "The system latency went from 200ms to 5 seconds on Tuesday.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-011",
|
||||
"input": "The new release should fix the login issue.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-012",
|
||||
"input": "I think therefore I am.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-013",
|
||||
"input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-014",
|
||||
"input": "The warehouse manager reported that inventory counts don't match the system.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-015",
|
||||
"input": "We've seen a 35% increase in customer complaints.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-016",
|
||||
"input": "The number of active users increased by 500%, from 4 to 2,001.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-017",
|
||||
"input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-018",
|
||||
"input": "The manufacturing team needs better quality control.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-019",
|
||||
"input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-020",
|
||||
"input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-021",
|
||||
"input": "Production output was 1,200 units last month and 1,180 units this month.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-022",
|
||||
"input": "The CFO reported that the company's cash position is healthy.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-023",
|
||||
"input": "We have enough funding to operate for 18 months.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-024",
|
||||
"input": "The new feature was deployed at 3am and user complaints tripled the next day.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-025",
|
||||
"input": "We need to launch a mobile app to capture market share.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-026",
|
||||
"input": "The system has been running for 90 days without failure since the migration.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-027",
|
||||
"input": "No one has submitted the required compliance report despite multiple reminders.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-028",
|
||||
"input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-029",
|
||||
"input": "We should implement the new CRM because our competitors have one.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-030",
|
||||
"input": "The server response time was acceptable last quarter but degraded this month.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-031",
|
||||
"input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-032",
|
||||
"input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-033",
|
||||
"input": "The database schema was changed on Friday but the reports are still working.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-034",
|
||||
"input": "Some team members say the new process is better while others say it's slower.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"investigate_contradiction",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-035",
|
||||
"input": "The application works fine on Chrome but not on Safari.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,633 +0,0 @@
|
||||
{
|
||||
"timestamp": "2026-07-31T18:23:11.669Z",
|
||||
"provider": "mock",
|
||||
"promptVersion": "v0.2",
|
||||
"casesRun": 35,
|
||||
"summary": {
|
||||
"schemaValidityRate": "100.0%",
|
||||
"classificationMatchRate": "48.6%",
|
||||
"reasoningModeMatchRate": "48.6%",
|
||||
"requiredConceptMatchRate": "0.0%",
|
||||
"unsupportedInferenceFailures": "0",
|
||||
"averageResponseDurationMs": "1",
|
||||
"fullPassRate": "48.6%"
|
||||
},
|
||||
"testCaseResults": [
|
||||
{
|
||||
"id": "tc-001",
|
||||
"input": "All customers cannot download their invoices.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 6,
|
||||
"actualPrimaryType": "fault_report",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-002",
|
||||
"input": "Some customers cannot download their invoices.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-003",
|
||||
"input": "Complaints increased by 35%.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "fault_report",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-004",
|
||||
"input": "Complaints increased by 35% while production increased by 40%.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "fault_report",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-005",
|
||||
"input": "Sales are falling.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "fault_report",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-006",
|
||||
"input": "Sales fell sharply immediately after the price increase.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-007",
|
||||
"input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-008",
|
||||
"input": "Revenue from the premium tier dropped while total revenue grew.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-009",
|
||||
"input": "We need to improve our customer retention rate.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-010",
|
||||
"input": "The system latency went from 200ms to 5 seconds on Tuesday.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-011",
|
||||
"input": "The new release should fix the login issue.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-012",
|
||||
"input": "I think therefore I am.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-013",
|
||||
"input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-014",
|
||||
"input": "The warehouse manager reported that inventory counts don't match the system.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-015",
|
||||
"input": "We've seen a 35% increase in customer complaints.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "fault_report",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-016",
|
||||
"input": "The number of active users increased by 500%, from 4 to 2,001.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-017",
|
||||
"input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-018",
|
||||
"input": "The manufacturing team needs better quality control.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-019",
|
||||
"input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "fault_report",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-020",
|
||||
"input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-021",
|
||||
"input": "Production output was 1,200 units last month and 1,180 units this month.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-022",
|
||||
"input": "The CFO reported that the company's cash position is healthy.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-023",
|
||||
"input": "We have enough funding to operate for 18 months.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-024",
|
||||
"input": "The new feature was deployed at 3am and user complaints tripled the next day.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "fault_report",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-025",
|
||||
"input": "We need to launch a mobile app to capture market share.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-026",
|
||||
"input": "The system has been running for 90 days without failure since the migration.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-027",
|
||||
"input": "No one has submitted the required compliance report despite multiple reminders.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "fault_report",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-028",
|
||||
"input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-029",
|
||||
"input": "We should implement the new CRM because our competitors have one.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-030",
|
||||
"input": "The server response time was acceptable last quarter but degraded this month.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-031",
|
||||
"input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "fault_report",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-032",
|
||||
"input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-033",
|
||||
"input": "The database schema was changed on Friday but the reports are still working.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-034",
|
||||
"input": "Some team members say the new process is better while others say it's slower.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"investigate_contradiction",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-035",
|
||||
"input": "The application works fine on Chrome but not on Safari.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,639 +0,0 @@
|
||||
{
|
||||
"timestamp": "2026-07-31T18:34:17.855Z",
|
||||
"provider": "mock",
|
||||
"promptVersion": "v0.2",
|
||||
"casesRun": 35,
|
||||
"summary": {
|
||||
"schemaValidityRate": "100.0%",
|
||||
"classificationMatchRate": "82.9%",
|
||||
"reasoningModeMatchRate": "51.4%",
|
||||
"requiredConceptMatchRate": "0.0%",
|
||||
"unsupportedInferenceFailures": "0",
|
||||
"averageResponseDurationMs": "1",
|
||||
"fullPassRate": "82.9%"
|
||||
},
|
||||
"testCaseResults": [
|
||||
{
|
||||
"id": "tc-001",
|
||||
"input": "All customers cannot download their invoices.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 5,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-002",
|
||||
"input": "Some customers cannot download their invoices.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 2,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-003",
|
||||
"input": "Complaints increased by 35%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-004",
|
||||
"input": "Complaints increased by 35% while production increased by 40%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-005",
|
||||
"input": "Sales are falling.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-006",
|
||||
"input": "Sales fell sharply immediately after the price increase.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-007",
|
||||
"input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"investigate_contradiction",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-008",
|
||||
"input": "Revenue from the premium tier dropped while total revenue grew.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-009",
|
||||
"input": "We need to improve our customer retention rate.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-010",
|
||||
"input": "The system latency went from 200ms to 5 seconds on Tuesday.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-011",
|
||||
"input": "The new release should fix the login issue.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-012",
|
||||
"input": "I think therefore I am.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-013",
|
||||
"input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-014",
|
||||
"input": "The warehouse manager reported that inventory counts don't match the system.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-015",
|
||||
"input": "We've seen a 35% increase in customer complaints.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-016",
|
||||
"input": "The number of active users increased by 500%, from 4 to 2,001.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-017",
|
||||
"input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-018",
|
||||
"input": "The manufacturing team needs better quality control.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-019",
|
||||
"input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-020",
|
||||
"input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-021",
|
||||
"input": "Production output was 1,200 units last month and 1,180 units this month.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-022",
|
||||
"input": "The CFO reported that the company's cash position is healthy.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-023",
|
||||
"input": "We have enough funding to operate for 18 months.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-024",
|
||||
"input": "The new feature was deployed at 3am and user complaints tripled the next day.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-025",
|
||||
"input": "We need to launch a mobile app to capture market share.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-026",
|
||||
"input": "The system has been running for 90 days without failure since the migration.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-027",
|
||||
"input": "No one has submitted the required compliance report despite multiple reminders.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-028",
|
||||
"input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-029",
|
||||
"input": "We should implement the new CRM because our competitors have one.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-030",
|
||||
"input": "The server response time was acceptable last quarter but degraded this month.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-031",
|
||||
"input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-032",
|
||||
"input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-033",
|
||||
"input": "The database schema was changed on Friday but the reports are still working.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-034",
|
||||
"input": "Some team members say the new process is better while others say it's slower.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-035",
|
||||
"input": "The application works fine on Chrome but not on Safari.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,644 +0,0 @@
|
||||
{
|
||||
"timestamp": "2026-07-31T18:59:37.227Z",
|
||||
"provider": "mock",
|
||||
"promptVersion": "v0.2",
|
||||
"casesRun": 35,
|
||||
"summary": {
|
||||
"schemaValidityRate": "100.0%",
|
||||
"classificationMatchRate": "100.0%",
|
||||
"reasoningModeMatchRate": "57.1%",
|
||||
"requiredConceptMatchRate": "0.0%",
|
||||
"unsupportedInferenceFailures": "0",
|
||||
"averageResponseDurationMs": "0",
|
||||
"fullPassRate": "100.0%"
|
||||
},
|
||||
"testCaseResults": [
|
||||
{
|
||||
"id": "tc-001",
|
||||
"input": "All customers cannot download their invoices.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 4,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-002",
|
||||
"input": "Some customers cannot download their invoices.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-003",
|
||||
"input": "Complaints increased by 35%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-004",
|
||||
"input": "Complaints increased by 35% while production increased by 40%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-005",
|
||||
"input": "Sales are falling.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-006",
|
||||
"input": "Sales fell sharply immediately after the price increase.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-007",
|
||||
"input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"investigate_contradiction",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-008",
|
||||
"input": "Revenue from the premium tier dropped while total revenue grew.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-009",
|
||||
"input": "We need to improve our customer retention rate.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-010",
|
||||
"input": "The system latency went from 200ms to 5 seconds on Tuesday.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-011",
|
||||
"input": "The new release should fix the login issue.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-012",
|
||||
"input": "I think therefore I am.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-013",
|
||||
"input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-014",
|
||||
"input": "The warehouse manager reported that inventory counts don't match the system.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-015",
|
||||
"input": "We've seen a 35% increase in customer complaints.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-016",
|
||||
"input": "The number of active users increased by 500%, from 4 to 2,001.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-017",
|
||||
"input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"investigate_contradiction",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-018",
|
||||
"input": "The manufacturing team needs better quality control.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-019",
|
||||
"input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-020",
|
||||
"input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-021",
|
||||
"input": "Production output was 1,200 units last month and 1,180 units this month.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-022",
|
||||
"input": "The CFO reported that the company's cash position is healthy.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-023",
|
||||
"input": "We have enough funding to operate for 18 months.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-024",
|
||||
"input": "The new feature was deployed at 3am and user complaints tripled the next day.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-025",
|
||||
"input": "We need to launch a mobile app to capture market share.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-026",
|
||||
"input": "The system has been running for 90 days without failure since the migration.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-027",
|
||||
"input": "No one has submitted the required compliance report despite multiple reminders.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-028",
|
||||
"input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-029",
|
||||
"input": "We should implement the new CRM because our competitors have one.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-030",
|
||||
"input": "The server response time was acceptable last quarter but degraded this month.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-031",
|
||||
"input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-032",
|
||||
"input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-033",
|
||||
"input": "The database schema was changed on Friday but the reports are still working.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-034",
|
||||
"input": "Some team members say the new process is better while others say it's slower.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-035",
|
||||
"input": "The application works fine on Chrome but not on Safari.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,644 +0,0 @@
|
||||
{
|
||||
"timestamp": "2026-07-31T19:00:09.575Z",
|
||||
"provider": "mock",
|
||||
"promptVersion": "v0.2",
|
||||
"casesRun": 35,
|
||||
"summary": {
|
||||
"schemaValidityRate": "100.0%",
|
||||
"classificationMatchRate": "100.0%",
|
||||
"reasoningModeMatchRate": "57.1%",
|
||||
"requiredConceptMatchRate": "0.0%",
|
||||
"unsupportedInferenceFailures": "0",
|
||||
"averageResponseDurationMs": "1",
|
||||
"fullPassRate": "100.0%"
|
||||
},
|
||||
"testCaseResults": [
|
||||
{
|
||||
"id": "tc-001",
|
||||
"input": "All customers cannot download their invoices.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 6,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-002",
|
||||
"input": "Some customers cannot download their invoices.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 2,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-003",
|
||||
"input": "Complaints increased by 35%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-004",
|
||||
"input": "Complaints increased by 35% while production increased by 40%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-005",
|
||||
"input": "Sales are falling.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-006",
|
||||
"input": "Sales fell sharply immediately after the price increase.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-007",
|
||||
"input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"investigate_contradiction",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-008",
|
||||
"input": "Revenue from the premium tier dropped while total revenue grew.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-009",
|
||||
"input": "We need to improve our customer retention rate.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-010",
|
||||
"input": "The system latency went from 200ms to 5 seconds on Tuesday.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-011",
|
||||
"input": "The new release should fix the login issue.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-012",
|
||||
"input": "I think therefore I am.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-013",
|
||||
"input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-014",
|
||||
"input": "The warehouse manager reported that inventory counts don't match the system.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-015",
|
||||
"input": "We've seen a 35% increase in customer complaints.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-016",
|
||||
"input": "The number of active users increased by 500%, from 4 to 2,001.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-017",
|
||||
"input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"investigate_contradiction",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-018",
|
||||
"input": "The manufacturing team needs better quality control.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-019",
|
||||
"input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-020",
|
||||
"input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-021",
|
||||
"input": "Production output was 1,200 units last month and 1,180 units this month.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-022",
|
||||
"input": "The CFO reported that the company's cash position is healthy.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-023",
|
||||
"input": "We have enough funding to operate for 18 months.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-024",
|
||||
"input": "The new feature was deployed at 3am and user complaints tripled the next day.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-025",
|
||||
"input": "We need to launch a mobile app to capture market share.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-026",
|
||||
"input": "The system has been running for 90 days without failure since the migration.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-027",
|
||||
"input": "No one has submitted the required compliance report despite multiple reminders.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-028",
|
||||
"input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-029",
|
||||
"input": "We should implement the new CRM because our competitors have one.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-030",
|
||||
"input": "The server response time was acceptable last quarter but degraded this month.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-031",
|
||||
"input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-032",
|
||||
"input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-033",
|
||||
"input": "The database schema was changed on Friday but the reports are still working.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-034",
|
||||
"input": "Some team members say the new process is better while others say it's slower.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-035",
|
||||
"input": "The application works fine on Chrome but not on Safari.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user