{
  "timestamp": "2026-08-01T06:11:41.538Z",
  "provider": "ollama-real",
  "promptVersion": "v0.2",
  "casesRun": 10,
  "summary": {
    "technical": {
      "schemaValidityRate": "0.0%",
      "classificationMatchRate": "0.0%",
      "nextQuestionPresentRate": "0.0%",
      "passRate": "0.0%"
    },
    "reasoningQuality": {
      "requiredConceptMatchRate": "100.0%",
      "unsupportedInferenceFailures": "0",
      "passRate": "0.0%"
    },
    "combinedPassRate": "0.0%",
    "averageResponseDurationMs": "19081"
  },
  "testCaseResults": [
    {
      "id": "diag-01",
      "input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
      "responseDurationMs": 19459,
      "actualPrimaryType": null,
      "actualReasoningModes": [],
      "technical": {
        "schemaValid": false,
        "classificationMatch": false,
        "reasoningModeMatch": false,
        "nextQuestionPresent": false,
        "pass": false,
        "errors": [
          "inputClassification: Required",
          "reconstruction: Required",
          "evidence: Required",
          "nextQuestion: Required"
        ]
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": true,
          "details": []
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": []
        },
        "pass": false
      }
    },
    {
      "id": "diag-02",
      "input": "Some customers reported that the new app crashes when uploading photos.",
      "responseDurationMs": 14693,
      "actualPrimaryType": null,
      "actualReasoningModes": [],
      "technical": {
        "schemaValid": false,
        "classificationMatch": false,
        "reasoningModeMatch": false,
        "nextQuestionPresent": false,
        "pass": false,
        "errors": [
          "Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download",
          "Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download"
        ]
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": true,
          "details": []
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": []
        },
        "pass": false
      }
    },
    {
      "id": "diag-03",
      "input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
      "responseDurationMs": 13516,
      "actualPrimaryType": null,
      "actualReasoningModes": [],
      "technical": {
        "schemaValid": false,
        "classificationMatch": false,
        "reasoningModeMatch": false,
        "nextQuestionPresent": false,
        "pass": false,
        "errors": [
          "inputClassification: Required",
          "reconstruction: Required",
          "evidence: Required",
          "nextQuestion: Required"
        ]
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": true,
          "details": []
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": []
        },
        "pass": false
      }
    },
    {
      "id": "diag-04",
      "input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
      "responseDurationMs": 48851,
      "actualPrimaryType": null,
      "actualReasoningModes": [],
      "technical": {
        "schemaValid": false,
        "classificationMatch": false,
        "reasoningModeMatch": false,
        "nextQuestionPresent": false,
        "pass": false,
        "errors": [
          "inputClassification: Required",
          "reconstruction: Required",
          "evidence: Required",
          "nextQuestion: Required"
        ]
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": true,
          "details": []
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": []
        },
        "pass": false
      }
    },
    {
      "id": "diag-05",
      "input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
      "responseDurationMs": 16508,
      "actualPrimaryType": null,
      "actualReasoningModes": [],
      "technical": {
        "schemaValid": false,
        "classificationMatch": false,
        "reasoningModeMatch": false,
        "nextQuestionPresent": false,
        "pass": false,
        "errors": [
          "inputClassification: Required",
          "reconstruction: Required",
          "evidence: Required",
          "nextQuestion: Required"
        ]
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": true,
          "details": []
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": []
        },
        "pass": false
      }
    },
    {
      "id": "diag-06",
      "input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
      "responseDurationMs": 15608,
      "actualPrimaryType": null,
      "actualReasoningModes": [],
      "technical": {
        "schemaValid": false,
        "classificationMatch": false,
        "reasoningModeMatch": false,
        "nextQuestionPresent": false,
        "pass": false,
        "errors": [
          "inputClassification: Required",
          "reconstruction: Required",
          "evidence: Required",
          "nextQuestion: Required"
        ]
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": true,
          "details": []
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": []
        },
        "pass": false
      }
    },
    {
      "id": "diag-07",
      "input": "A user claimed that our pricing model is too complex for small businesses.",
      "responseDurationMs": 15750,
      "actualPrimaryType": null,
      "actualReasoningModes": [],
      "technical": {
        "schemaValid": false,
        "classificationMatch": false,
        "reasoningModeMatch": false,
        "nextQuestionPresent": false,
        "pass": false,
        "errors": [
          "inputClassification: Required",
          "reconstruction: Required",
          "evidence: Required",
          "nextQuestion: Required"
        ]
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": true,
          "details": []
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": []
        },
        "pass": false
      }
    },
    {
      "id": "diag-08",
      "input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
      "responseDurationMs": 15383,
      "actualPrimaryType": null,
      "actualReasoningModes": [],
      "technical": {
        "schemaValid": false,
        "classificationMatch": false,
        "reasoningModeMatch": false,
        "nextQuestionPresent": false,
        "pass": false,
        "errors": [
          "inputClassification: Required",
          "reconstruction: Required",
          "evidence: Required",
          "nextQuestion: Required"
        ]
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": true,
          "details": []
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": []
        },
        "pass": false
      }
    },
    {
      "id": "diag-09",
      "input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
      "responseDurationMs": 15643,
      "actualPrimaryType": null,
      "actualReasoningModes": [],
      "technical": {
        "schemaValid": false,
        "classificationMatch": false,
        "reasoningModeMatch": false,
        "nextQuestionPresent": false,
        "pass": false,
        "errors": [
          "inputClassification: Required",
          "reconstruction: Required",
          "evidence: Required",
          "nextQuestion: Required"
        ]
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": true,
          "details": []
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": []
        },
        "pass": false
      }
    },
    {
      "id": "diag-10",
      "input": "Some complaints involve production issues, but others say the delivery team is slow.",
      "responseDurationMs": 15401,
      "actualPrimaryType": null,
      "actualReasoningModes": [],
      "technical": {
        "schemaValid": false,
        "classificationMatch": false,
        "reasoningModeMatch": false,
        "nextQuestionPresent": false,
        "pass": false,
        "errors": [
          "inputClassification: Required",
          "reconstruction: Required",
          "evidence: Required",
          "nextQuestion: Required"
        ]
      },
      "reasoningQuality": {
        "requiredConcepts": {
          "pass": true,
          "details": []
        },
        "unsupportedInferencesAbsent": {
          "pass": true,
          "details": []
        },
        "pass": false
      }
    }
  ]
}