{
  "timestamp": "2026-08-01T06:01:56.070Z",
  "provider": "mock",
  "promptVersion": "v0.2",
  "casesRun": 10,
  "summary": {
    "technical": {
      "schemaValidityRate": "100.0%",
      "classificationMatchRate": "80.0%",
      "nextQuestionPresentRate": "100.0%",
      "passRate": "80.0%"
    },
    "reasoningQuality": {
      "requiredConceptMatchRate": "0.0%",
      "unsupportedInferenceFailures": "0",
      "passRate": "0.0%"
    },
    "combinedPassRate": "0.0%",
    "averageResponseDurationMs": "2"
  },
  "testCaseResults": [
    {
      "id": "diag-01",
      "input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
      "responseDurationMs": 6,
      "actualPrimaryType": "unexplained_change",
      "actualReasoningModes": [
        "establish_baseline",
        "identify_difference"
      ],
      "technical": {
        "schemaValid": true,
        "classificationMatch": true,
        "reasoningModeMatch": true,
        "nextQuestionPresent": true,
        "pass": true,
        "errors": []
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": false,
          "details": [
            {
              "concept": "complaints",
              "found": true
            },
            {
              "concept": "warehouse",
              "found": false
            },
            {
              "concept": "baseline comparison",
              "found": false
            }
          ]
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": [
            {
              "concept": "quality issue",
              "absent": true
            },
            {
              "concept": "staff turnover",
              "absent": true
            },
            {
              "concept": "training gap",
              "absent": true
            }
          ]
        },
        "pass": false
      }
    },
    {
      "id": "diag-02",
      "input": "Some customers reported that the new app crashes when uploading photos.",
      "responseDurationMs": 2,
      "actualPrimaryType": "observed_problem",
      "actualReasoningModes": [
        "identify_difference"
      ],
      "technical": {
        "schemaValid": true,
        "classificationMatch": true,
        "reasoningModeMatch": true,
        "nextQuestionPresent": true,
        "pass": true,
        "errors": []
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": false,
          "details": [
            {
              "concept": "app crashes",
              "found": false
            },
            {
              "concept": "photo upload",
              "found": false
            },
            {
              "concept": "some customers",
              "found": false
            }
          ]
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": [
            {
              "concept": "all users affected",
              "absent": true
            },
            {
              "concept": "server-side bug",
              "absent": true
            },
            {
              "concept": "Android only",
              "absent": true
            }
          ]
        },
        "pass": false
      }
    },
    {
      "id": "diag-03",
      "input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
      "responseDurationMs": 1,
      "actualPrimaryType": "causal_claim",
      "actualReasoningModes": [
        "establish_baseline",
        "identify_difference"
      ],
      "technical": {
        "schemaValid": true,
        "classificationMatch": false,
        "reasoningModeMatch": true,
        "nextQuestionPresent": true,
        "pass": false,
        "errors": []
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": false,
          "details": [
            {
              "concept": "sales decline",
              "found": false
            },
            {
              "concept": "price increase",
              "found": false
            },
            {
              "concept": "revenue increase",
              "found": false
            },
            {
              "concept": "CFO report",
              "found": false
            }
          ]
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": [
            {
              "concept": "price was set too high",
              "absent": true
            },
            {
              "concept": "competitors gained market share",
              "absent": true
            },
            {
              "concept": "revenue data is wrong",
              "absent": true
            }
          ]
        },
        "pass": false
      }
    },
    {
      "id": "diag-04",
      "input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
      "responseDurationMs": 0,
      "actualPrimaryType": "decision_request",
      "actualReasoningModes": [
        "identify_difference",
        "decision_support",
        "identify_missing_information"
      ],
      "technical": {
        "schemaValid": true,
        "classificationMatch": true,
        "reasoningModeMatch": true,
        "nextQuestionPresent": true,
        "pass": true,
        "errors": []
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": false,
          "details": [
            {
              "concept": "marketplace app",
              "found": false
            },
            {
              "concept": "Southeast Asia",
              "found": false
            },
            {
              "concept": "competitor gap",
              "found": false
            }
          ]
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": [
            {
              "concept": "this will definitely succeed",
              "absent": true
            },
            {
              "concept": "we have the resources",
              "absent": true
            },
            {
              "concept": "competitors are struggling",
              "absent": true
            }
          ]
        },
        "pass": false
      }
    },
    {
      "id": "diag-05",
      "input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
      "responseDurationMs": 1,
      "actualPrimaryType": "unexplained_change",
      "actualReasoningModes": [
        "identify_difference",
        "establish_baseline",
        "validate_measurement"
      ],
      "technical": {
        "schemaValid": true,
        "classificationMatch": true,
        "reasoningModeMatch": true,
        "nextQuestionPresent": true,
        "pass": true,
        "errors": []
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": false,
          "details": [
            {
              "concept": "supplier change",
              "found": false
            },
            {
              "concept": "three months ago",
              "found": false
            },
            {
              "concept": "same defect rate",
              "found": false
            }
          ]
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": [
            {
              "concept": "new supplier is worse",
              "absent": true
            },
            {
              "concept": "old supplier was better",
              "absent": true
            },
            {
              "concept": "quality process is broken",
              "absent": true
            }
          ]
        },
        "pass": false
      }
    },
    {
      "id": "diag-06",
      "input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
      "responseDurationMs": 1,
      "actualPrimaryType": "observed_problem",
      "actualReasoningModes": [
        "identify_difference"
      ],
      "technical": {
        "schemaValid": true,
        "classificationMatch": false,
        "reasoningModeMatch": false,
        "nextQuestionPresent": true,
        "pass": false,
        "errors": []
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": false,
          "details": [
            {
              "concept": "completion rate",
              "found": false
            },
            {
              "concept": "45%",
              "found": false
            },
            {
              "concept": "62%",
              "found": false
            },
            {
              "concept": "onboarding",
              "found": false
            }
          ]
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": [
            {
              "concept": "all improvements are due to the redesign",
              "absent": true
            },
            {
              "concept": "the old flow was bad",
              "absent": true
            },
            {
              "concept": "users prefer the new design",
              "absent": true
            }
          ]
        },
        "pass": false
      }
    },
    {
      "id": "diag-07",
      "input": "A user claimed that our pricing model is too complex for small businesses.",
      "responseDurationMs": 1,
      "actualPrimaryType": "reported_claim",
      "actualReasoningModes": [
        "identify_difference",
        "validate_claim"
      ],
      "technical": {
        "schemaValid": true,
        "classificationMatch": true,
        "reasoningModeMatch": true,
        "nextQuestionPresent": true,
        "pass": true,
        "errors": []
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": false,
          "details": [
            {
              "concept": "pricing complexity",
              "found": false
            },
            {
              "concept": "small business",
              "found": false
            },
            {
              "concept": "user claim",
              "found": false
            }
          ]
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": [
            {
              "concept": "the pricing is actually complex",
              "absent": true
            },
            {
              "concept": "other small businesses agree",
              "absent": true
            },
            {
              "concept": "we should simplify pricing",
              "absent": true
            }
          ]
        },
        "pass": false
      }
    },
    {
      "id": "diag-08",
      "input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
      "responseDurationMs": 1,
      "actualPrimaryType": "ambiguous_statement",
      "actualReasoningModes": [
        "identify_difference",
        "clarify_meaning"
      ],
      "technical": {
        "schemaValid": true,
        "classificationMatch": true,
        "reasoningModeMatch": true,
        "nextQuestionPresent": true,
        "pass": true,
        "errors": []
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": false,
          "details": [
            {
              "concept": "philosophical",
              "found": false
            },
            {
              "concept": "ambiguous",
              "found": true
            },
            {
              "concept": "meaning clarification",
              "found": false
            }
          ]
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": [
            {
              "concept": "the phrase was wrong",
              "absent": true
            },
            {
              "concept": "the colleague is hostile",
              "absent": true
            },
            {
              "concept": "we should avoid philosophical language",
              "absent": true
            }
          ]
        },
        "pass": false
      }
    },
    {
      "id": "diag-09",
      "input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
      "responseDurationMs": 1,
      "actualPrimaryType": "causal_claim",
      "actualReasoningModes": [
        "establish_baseline",
        "identify_difference"
      ],
      "technical": {
        "schemaValid": true,
        "classificationMatch": true,
        "reasoningModeMatch": true,
        "nextQuestionPresent": true,
        "pass": true,
        "errors": []
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": false,
          "details": [
            {
              "concept": "deployment",
              "found": false
            },
            {
              "concept": "complaint volume increase",
              "found": false
            },
            {
              "concept": "tripled",
              "found": false
            },
            {
              "concept": "47 cases",
              "found": false
            }
          ]
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": [
            {
              "concept": "the deployment caused the complaints",
              "absent": true
            },
            {
              "concept": "the bug report was insufficient",
              "absent": true
            },
            {
              "concept": "rollback is needed",
              "absent": true
            }
          ]
        },
        "pass": false
      }
    },
    {
      "id": "diag-10",
      "input": "Some complaints involve production issues, but others say the delivery team is slow.",
      "responseDurationMs": 1,
      "actualPrimaryType": "observed_problem",
      "actualReasoningModes": [
        "establish_baseline",
        "identify_difference"
      ],
      "technical": {
        "schemaValid": true,
        "classificationMatch": true,
        "reasoningModeMatch": true,
        "nextQuestionPresent": true,
        "pass": true,
        "errors": []
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": false,
          "details": [
            {
              "concept": "production issues",
              "found": false
            },
            {
              "concept": "delivery speed",
              "found": false
            },
            {
              "concept": "complaint types",
              "found": false
            }
          ]
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": [
            {
              "concept": "production is worse than delivery",
              "absent": true
            },
            {
              "concept": "the delivery team needs training",
              "absent": true
            },
            {
              "concept": "both teams are underperforming equally",
              "absent": true
            }
          ]
        },
        "pass": false
      }
    }
  ]
}