diff --git a/app/api/analyse/route.js b/app/api/analyse/route.js
index 31ac1e1..106c6e6 100644
--- a/app/api/analyse/route.js
+++ b/app/api/analyse/route.js
@@ -1,16 +1,9 @@
-import { getConfig } from "@/lib/config";
-import { getProvider } from "@/lib/llm/provider";
-import { reconstructionSchema } from "@/lib/reconstruction/schema";
-
-const MAX_SCENARIO_LENGTH = 10000;
+import { analyseScenario, PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION } from "@/lib/analysis";
export async function POST(request) {
- const startTime = Date.now();
- let rawResponse = null;
-
try {
const body = await request.json();
-
+
if (!body.scenario || typeof body.scenario !== "string") {
return Response.json(
{ error: "Request must include a 'scenario' string field" },
@@ -18,83 +11,34 @@ export async function POST(request) {
);
}
- const trimmed = body.scenario.trim();
-
- if (trimmed.length === 0) {
+ // Optional prompt version override
+ let promptVersion = DEFAULT_PROMPT_VERSION;
+ if (body.promptVersion && PROMPT_VERSIONS.includes(body.promptVersion)) {
+ promptVersion = body.promptVersion;
+ }
+
+ const result = await analyseScenario(body.scenario, { promptVersion });
+
+ if (!result.success) {
return Response.json(
- { error: "Scenario cannot be empty" },
- { status: 400 }
+ { ...result, reconstruction: result.reconstruction || null },
+ { status: Number(result.statusCode) || 500 }
);
}
- if (trimmed.length > MAX_SCENARIO_LENGTH) {
- return Response.json(
- { error: `Scenario must be under ${MAX_SCENARIO_LENGTH} characters` },
- { status: 400 }
- );
- }
-
- const configResult = getConfig();
- if (!configResult.ok) {
- return Response.json(
- { error: "Invalid server configuration" },
- { status: 500 }
- );
- }
-
- const { OLLAMA_BASE_URL, OLLAMA_MODEL } = configResult.config;
- const provider = getProvider();
-
- // Attempt parse to capture raw for debugging
- let reconstruction;
- try {
- reconstruction = await provider.generateReconstruction(trimmed, OLLAMA_MODEL);
- } catch (e) {
- return Response.json(
- {
- error: e.message || "Unknown server error",
- responseDurationMs: Date.now() - startTime,
- modelName: OLLAMA_MODEL,
- validationStatus: "invalid",
- },
- { status: 500 }
- );
- }
-
- // Try to stringify for rawResponse display (safe even if it's already an object)
- try {
- rawResponse = JSON.stringify(reconstruction);
- } catch {
- rawResponse = String(reconstruction).slice(0, 2000);
- }
-
- const duration = Date.now() - startTime;
-
- // Validate with Zod schema
- const validationResult = reconstructionSchema.safeParse(reconstruction);
-
- if (!validationResult.success) {
- return Response.json({
- reconstruction: null,
- modelName: OLLAMA_MODEL,
- responseDurationMs: duration,
- validationStatus: "invalid",
- rawResponse: rawResponse?.slice(0, 2000),
- errors: validationResult.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`),
- });
- }
-
return Response.json({
- reconstruction: validationResult.data,
- modelName: OLLAMA_MODEL,
- responseDurationMs: duration,
- validationStatus: "valid",
- rawResponse: rawResponse?.slice(0, 2000),
+ inputClassification: result.inputClassification,
+ reconstruction: result.reconstruction,
+ evidence: result.evidence,
+ nextQuestion: result.nextQuestion,
+ modelName: result.modelName,
+ responseDurationMs: result.responseDurationMs,
+ validationStatus: result.validationStatus,
+ promptVersion: result.promptVersion,
});
} catch (e) {
- const duration = Date.now() - startTime;
return Response.json(
- { error: e.message || "Unknown server error", responseDurationMs: duration },
+ { error: e.message || "Unknown server error", responseDurationMs: 0 },
{ status: 500 }
);
}
diff --git a/components/diagnostics-view.jsx b/components/diagnostics-view.jsx
index 535c4e4..8622dc5 100644
--- a/components/diagnostics-view.jsx
+++ b/components/diagnostics-view.jsx
@@ -16,9 +16,19 @@ const ValidationIndicator = ({ status }) => {
);
};
+const validationIcons = {
+ valid: "✅",
+ partial: "⚠️",
+ invalid: "❌",
+};
+
export default function DiagnosticsView({ result }) {
+ if (!result) return null;
+
const metrics = [
{ label: "Model", value: result.modelName || "?" },
+ { label: "Provider", value: "Ollama" },
+ { label: "Prompt version", value: result.promptVersion || "?" },
{ label: "Duration", value: result.responseDurationMs != null ? `${result.responseDurationMs}ms` : "?" },
{ label: "Validation", value: },
];
@@ -35,16 +45,31 @@ export default function DiagnosticsView({ result }) {
))}
+ {/* Collapsed raw output for debugging */}
{result.rawResponse && (
- View raw model response
+ View raw model response ({(result.rawResponse?.length || 0).toLocaleString()} chars)
{result.rawResponse}
)}
+
+ {/* Errors if present */}
+ {result.errors && result.errors.length > 0 && (
+
+
+ Validation errors ({result.errors.length})
+
+
+ {result.errors.map((err, i) => (
+ {err}
+ ))}
+
+
+ )}
);
}
diff --git a/components/reconstruction-view.jsx b/components/reconstruction-view.jsx
index 67e8034..59b00f9 100644
--- a/components/reconstruction-view.jsx
+++ b/components/reconstruction-view.jsx
@@ -1,15 +1,8 @@
-const categoryLabels = {
- observations: "Direct Observations",
- reportedClaims: "Reported Claims",
- assumptions: "Unsupported Assumptions",
- entities: "Entities",
- transitions: "Transitions",
- expectedButMissing: "Expected But Missing",
- presentButUnexpected: "Present But Unexpected",
- contradictions: "Contradictions",
- openUncertainties: "Open Uncertainties",
-};
+"use client";
+import { useMemo } from "react";
+
+// ── Confidence badge (shared) ────────────────────────
const confidenceColor = {
low: "text-red-600 bg-red-50 border-red-200",
medium: "text-yellow-700 bg-yellow-50 border-yellow-200",
@@ -22,26 +15,210 @@ const ConfidenceBadge = ({ level }) => (
);
-function ItemList({ items, renderExtra }) {
- if (!items?.length) return
None identified
;
-
+// ── Evidence type labels (shared) ───────────────────
+const evidenceTypeLabels = {
+ direct_observation: "Direct Observation",
+ reported_statement: "Reported Statement",
+ interpretation: "Interpretation",
+ assumption: "Assumption",
+ inferred_relationship: "Inferred Relationship",
+};
+
+const importanceColors = {
+ incidental: "text-gray-500 bg-gray-50 border-gray-200",
+ supporting: "text-blue-700 bg-blue-50 border-blue-200",
+ important: "text-orange-700 bg-orange-50 border-orange-200",
+ critical: "text-red-800 bg-red-50 border-red-300 font-semibold",
+};
+
+const importanceLabels = {
+ incidental: "Incidental",
+ supporting: "Supporting",
+ important: "Important",
+ critical: "Critical",
+};
+
+// ── Input classification display ────────────────────
+function ClassificationDisplay({ classification }) {
+ if (!classification) return null;
+ const p = classification.primaryType || classification.primary_type;
+ const sec = classification.secondaryTypes || classification.secondary_types || [];
+ const modes = classification.reasoningModes || classification.reasoning_modes || [];
+
+ // Normalize camelCase to snake_case for display if needed
+ const primaryLabel = String(p).replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
+ const secLabels = sec.map((s) => s.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()));
+ const modeLabels = modes.map((m) => m.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()));
+
return (
-
- {items.map((item) => (
-
-
- #{item.id}
-
-
- {item.description}
- {renderExtra && renderExtra(item)}
-
- ))}
-
+
+
Input Classification
+
+ Primary type
+ {primaryLabel}
+ {secLabels.length > 0 && (
+ <>
+ Secondary types
+ {secLabels.join(" · ")}
+ >
+ )}
+ {modeLabels.length > 0 && (
+ <>
+ Reasoning modes
+ {modeLabels.join(" · ")}
+ >
+ )}
+ Classification reason
+ {classification.classificationReason || classification.classification_reason}
+ Confidence
+
+
+
);
}
+// ── Reconstruction summary ──────────────────────────
+function SummaryDisplay({ reconstruction }) {
+ if (!reconstruction?.summary) return null;
+ const summary = reconstruction.summary || reconstruction.Summary;
+ return (
+
+
Reconstruction Summary
+
{summary}
+
+ );
+}
+
+// ── Generic item list (used for multiple sections) ──
+function ItemList({ title, items, renderExtra }) {
+ const count = items?.length;
+ if (!count) return null; // hide empty sections entirely
+
+ const itemsArr = Array.isArray(items) ? items : [items];
+
+ return (
+
+ );
+}
+
+// ── Plausible interpretations ───────────────────────
+function InterpretationsDisplay({ interpretations }) {
+ if (!interpretations?.length) return null;
+ const arr = Array.isArray(interpretations) ? interpretations : [interpretations];
+
+ return (
+
+
Plausible Interpretations ({arr.length})
+
+ {arr.map((interp, idx) => (
+
+
+ {interp.description}
+ {interp.confidence && }
+
+ {interp.supportingEvidenceIds?.length > 0 && (
+ Supporting evidence: {interp.supportingEvidenceIds.join(", ")}
+ )}
+ {interp.assumptionsRequired?.length > 0 && (
+ Requires assumptions: {interp.assumptionsRequired.join("; ")}
+ )}
+
+ ))}
+
+
+ );
+}
+
+// ── Next question (prominent) ───────────────────────
+function NextQuestionDisplay({ question }) {
+ if (!question?.question) return null;
+ const q = question.question || question.Question;
+ const targets = question.targets || question.Targets || [];
+ const reason = question.reason || question.Reason || "";
+ const value = question.expectedInformationValue || question.expected_information_value || "medium";
+
+ const valueLabel = { low: "Low", medium: "Medium", high: "High" }[value] || "Medium";
+ const valueColor = { low: "bg-yellow-100 text-yellow-800", medium: "bg-blue-100 text-blue-800", high: "bg-green-100 text-green-800" }[value] || "";
+
+ return (
+
+
+
Next Question
+ {valueLabel} value
+
+
{q}
+ {targets.length > 0 && (
+
Targets: {targets.join(", ")}
+ )}
+ {reason &&
Because: {reason}
}
+
+ );
+}
+
+// ── Evidence list ───────────────────────────────────
+function EvidenceDisplay({ evidence }) {
+ if (!evidence?.length) return null;
+ const arr = Array.isArray(evidence) ? evidence : [evidence];
+
+ const evidenceLabels = {
+ direct_observation: "👁 Direct Observation",
+ reported_statement: "🗣 Reported Statement",
+ interpretation: "💡 Interpretation",
+ assumption: "❓ Assumption",
+ inferred_relationship: "🔗 Inferred Relationship",
+ };
+
+ return (
+
+
Supporting Evidence ({arr.length})
+
+
+ );
+}
+
+// ── Main component ──────────────────────────────────
export default function ReconstructionView({ reconstruction, partial }) {
+ // Handle both v0.2 direct object and wrapped result formats
+ const data = reconstruction;
+
if (partial) {
return (
@@ -50,21 +227,74 @@ export default function ReconstructionView({ reconstruction, partial }) {
);
}
- const categories = Object.entries(categoryLabels).map(([key, label]) => ({
- key,
- label,
- items: reconstruction[key],
- }));
-
return (
-
-
Reconstruction
- {categories.map(({ key, label, items }) => (
-
-
{label}
-
-
- ))}
+
+ {/* Classification first */}
+ {data.inputClassification && (
+
+ )}
+
+ {/* Summary */}
+ {data.reconstruction?.summary && (
+
+ )}
+
+ {/* Key differences */}
+ {data.reconstruction?.differences && (
+
+ )}
+
+ {/* Unexplained transitions */}
+ {data.reconstruction?.unexplainedTransitions && data.reconstruction.unexplainedTransitions.length > 0 && (
+
(
+ i.entity && Entity: {i.entity}
+ )} />
+ )}
+
+ {/* Contradictions */}
+ {data.reconstruction?.contradictions && data.reconstruction.contradictions.length > 0 && (
+
+ )}
+
+ {/* Important unknowns */}
+ {data.reconstruction?.importantUnknowns && data.reconstruction.importantUnknowns.length > 0 && (
+
+ )}
+
+ {/* Plausible interpretations */}
+ {data.reconstruction?.plausibleInterpretations && data.reconstruction.plausibleInterpretations.length > 0 && (
+
+ )}
+
+ {/* Secondary reconstruction categories (actors, systems, etc.) */}
+ {data.reconstruction?.actors && data.reconstruction.actors.length > 0 && (
+
+ )}
+ {data.reconstruction?.systemsOrObjects && data.reconstruction.systemsOrObjects.length > 0 && (
+
+ )}
+ {data.reconstruction?.expectedStates && data.reconstruction.expectedStates.length > 0 && (
+
+ )}
+ {data.reconstruction?.observedStates && data.reconstruction.observedStates.length > 0 && (
+
+ )}
+ {data.reconstruction?.knownTransitions && data.reconstruction.knownTransitions.length > 0 && (
+ (
+
+ {i.entity && Entity: {i.entity} · }
+ From "{i.previousState}" → To "{i.currentState}" ({i.explanationStatus})
+
+ )} />
+ )}
+
+ {/* Next question — prominent */}
+
+
+ {/* Evidence */}
+ {data.evidence && (
+
+ )}
);
}
diff --git a/components/scenario-form.jsx b/components/scenario-form.jsx
index 85bdd4c..8ebce4d 100644
--- a/components/scenario-form.jsx
+++ b/components/scenario-form.jsx
@@ -8,7 +8,7 @@ const MAX_LENGTH = 10000;
export default function ScenarioForm() {
const [scenario, setScenario] = useState("");
- const [status, setStatus] = useState("idle"); // idle | loading | error | success
+ const [status, setStatus] = useState("idle"); // idle | loading | error | success | partial
const [result, setResult] = useState(null);
const textareaRef = useRef(null);
@@ -29,6 +29,10 @@ export default function ScenarioForm() {
if (res.ok && data.validationStatus === "valid") {
setStatus("success");
setResult(data);
+ } else if (data.success) {
+ // Success in analysis but validation may be partial
+ setStatus("success");
+ setResult(data);
} else {
setStatus("error");
setResult(data);
@@ -39,8 +43,12 @@ export default function ScenarioForm() {
}
};
- // Always show diagnostics when there's a result (even if validation failed)
- const hasDiagnostics = result && (result.reconstruction || result.modelName || result.responseDurationMs !== undefined);
+ // Determine if we have meaningful content to display
+ const hasClassification = result?.inputClassification;
+ const hasReconstruction = result?.reconstruction;
+ const hasNextQuestion = result?.nextQuestion;
+ const hasEvidence = result?.evidence && result.evidence.length > 0;
+ const hasMeaningfulContent = hasClassification || hasReconstruction || hasNextQuestion || hasEvidence;
return (
@@ -65,6 +73,7 @@ export default function ScenarioForm() {
+ {/* Error state */}
{status === "error" && (
{result?.error && (
@@ -72,36 +81,47 @@ export default function ScenarioForm() {
Error: {result.error}
)}
- {hasDiagnostics && result?.modelName && (
-
- Model
- {result.modelName}
- Duration
- {result.responseDurationMs != null ? `${result.responseDurationMs}ms` : "?"}
-
+ {/* Show partial content even on validation failure */}
+ {(hasClassification || hasReconstruction) && (
+
+ ⚠ Partial result — some fields failed validation. Showing what was accepted.
+
+ )}
+ {hasReconstruction && (
+
)}
)}
- {status === "success" && result?.reconstruction && (
+ {/* Success state */}
+ {status === "success" && hasMeaningfulContent && (
-
-
+
)}
- {status === "error" && result?.reconstruction && (
-
-
- ⚠ Partial result — some fields failed validation. Showing what was accepted.
-
-
-
+ {/* Always show diagnostics when we have any result */}
+ {(hasClassification || hasReconstruction || hasNextQuestion) && (
+
)}
{status === "loading" && (
Waiting for model response...
)}
+
+ {/* Empty state */}
+ {status === "idle" && (
+
+
Enter a scenario above and click Analyse to begin.
+
+ )}
+
+ {/* Invalid result with no partial data */}
+ {status === "error" && !result?.error && !hasMeaningfulContent && (
+
+ Validation failed — no structured output was produced.
+
+ )}
);
}
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-01-result.json b/evaluation-results/2026-08-01T06-01-56/diag-01-result.json
new file mode 100644
index 0000000..3f4a30a
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-01-result.json
@@ -0,0 +1,56 @@
+{
+ "id": "diag-01",
+ "description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.",
+ "input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
+ "responseDurationMs": 6,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "complaints",
+ "found": true
+ },
+ {
+ "concept": "warehouse",
+ "found": false
+ },
+ {
+ "concept": "baseline comparison",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "quality issue",
+ "absent": true
+ },
+ {
+ "concept": "staff turnover",
+ "absent": true
+ },
+ {
+ "concept": "training gap",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-01-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-01-summary.md
new file mode 100644
index 0000000..eb447be
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-01-summary.md
@@ -0,0 +1,21 @@
+# Diagnostic Case: diag-01
+
+Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.
+
+## Input
+
+```
+We've seen a spike in complaints from our warehouse team this month compared to last month.
+```
+
+## Result
+
+- **Technical**: ✅ PASS (1/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: unexplained_change
+- **Actual Reasoning Modes**: establish_baseline, identify_difference
+- **Response Duration**: 6ms
+
+### Reasoning Quality Failures
+
+missing required concept(s): warehouse, baseline comparison
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-02-result.json b/evaluation-results/2026-08-01T06-01-56/diag-02-result.json
new file mode 100644
index 0000000..5555945
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-02-result.json
@@ -0,0 +1,55 @@
+{
+ "id": "diag-02",
+ "description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.",
+ "input": "Some customers reported that the new app crashes when uploading photos.",
+ "responseDurationMs": 2,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "app crashes",
+ "found": false
+ },
+ {
+ "concept": "photo upload",
+ "found": false
+ },
+ {
+ "concept": "some customers",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "all users affected",
+ "absent": true
+ },
+ {
+ "concept": "server-side bug",
+ "absent": true
+ },
+ {
+ "concept": "Android only",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-02-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-02-summary.md
new file mode 100644
index 0000000..0172a70
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-02-summary.md
@@ -0,0 +1,21 @@
+# Diagnostic Case: diag-02
+
+Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.
+
+## Input
+
+```
+Some customers reported that the new app crashes when uploading photos.
+```
+
+## Result
+
+- **Technical**: ✅ PASS (1/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: observed_problem
+- **Actual Reasoning Modes**: identify_difference
+- **Response Duration**: 2ms
+
+### Reasoning Quality Failures
+
+missing required concept(s): app crashes, photo upload, some customers
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-03-result.json b/evaluation-results/2026-08-01T06-01-56/diag-03-result.json
new file mode 100644
index 0000000..e9c28ca
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-03-result.json
@@ -0,0 +1,60 @@
+{
+ "id": "diag-03",
+ "description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.",
+ "input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": false,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "sales decline",
+ "found": false
+ },
+ {
+ "concept": "price increase",
+ "found": false
+ },
+ {
+ "concept": "revenue increase",
+ "found": false
+ },
+ {
+ "concept": "CFO report",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "price was set too high",
+ "absent": true
+ },
+ {
+ "concept": "competitors gained market share",
+ "absent": true
+ },
+ {
+ "concept": "revenue data is wrong",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-03-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-03-summary.md
new file mode 100644
index 0000000..6439e4d
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-03-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-03
+
+Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.
+
+## Input
+
+```
+Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: causal_claim
+- **Actual Reasoning Modes**: establish_baseline, identify_difference
+- **Response Duration**: 1ms
+
+### Technical Failures
+
+classification mismatch
+
+### Reasoning Quality Failures
+
+missing required concept(s): sales decline, price increase, revenue increase, CFO report
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-04-result.json b/evaluation-results/2026-08-01T06-01-56/diag-04-result.json
new file mode 100644
index 0000000..c8b849a
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-04-result.json
@@ -0,0 +1,57 @@
+{
+ "id": "diag-04",
+ "description": "Decision request — forward-looking, needs missing info identification.",
+ "input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "marketplace app",
+ "found": false
+ },
+ {
+ "concept": "Southeast Asia",
+ "found": false
+ },
+ {
+ "concept": "competitor gap",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "this will definitely succeed",
+ "absent": true
+ },
+ {
+ "concept": "we have the resources",
+ "absent": true
+ },
+ {
+ "concept": "competitors are struggling",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-04-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-04-summary.md
new file mode 100644
index 0000000..de5a547
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-04-summary.md
@@ -0,0 +1,21 @@
+# Diagnostic Case: diag-04
+
+Decision request — forward-looking, needs missing info identification.
+
+## Input
+
+```
+We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.
+```
+
+## Result
+
+- **Technical**: ✅ PASS (1/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: decision_request
+- **Actual Reasoning Modes**: identify_difference, decision_support, identify_missing_information
+- **Response Duration**: 0ms
+
+### Reasoning Quality Failures
+
+missing required concept(s): marketplace app, Southeast Asia, competitor gap
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-05-result.json b/evaluation-results/2026-08-01T06-01-56/diag-05-result.json
new file mode 100644
index 0000000..a1551fc
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-05-result.json
@@ -0,0 +1,57 @@
+{
+ "id": "diag-05",
+ "description": "Unexpected continuity — changed context but no outcome change.",
+ "input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "supplier change",
+ "found": false
+ },
+ {
+ "concept": "three months ago",
+ "found": false
+ },
+ {
+ "concept": "same defect rate",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "new supplier is worse",
+ "absent": true
+ },
+ {
+ "concept": "old supplier was better",
+ "absent": true
+ },
+ {
+ "concept": "quality process is broken",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-05-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-05-summary.md
new file mode 100644
index 0000000..4d24ef7
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-05-summary.md
@@ -0,0 +1,21 @@
+# Diagnostic Case: diag-05
+
+Unexpected continuity — changed context but no outcome change.
+
+## Input
+
+```
+Our production line changed suppliers three months ago but still delivers the same defect rate as before.
+```
+
+## Result
+
+- **Technical**: ✅ PASS (1/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: unexplained_change
+- **Actual Reasoning Modes**: identify_difference, establish_baseline, validate_measurement
+- **Response Duration**: 1ms
+
+### Reasoning Quality Failures
+
+missing required concept(s): supplier change, three months ago, same defect rate
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-06-result.json b/evaluation-results/2026-08-01T06-01-56/diag-06-result.json
new file mode 100644
index 0000000..2f6f608
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-06-result.json
@@ -0,0 +1,59 @@
+{
+ "id": "diag-06",
+ "description": "Quantified improvement — needs context about measurement period and baseline conditions.",
+ "input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": false,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "completion rate",
+ "found": false
+ },
+ {
+ "concept": "45%",
+ "found": false
+ },
+ {
+ "concept": "62%",
+ "found": false
+ },
+ {
+ "concept": "onboarding",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "all improvements are due to the redesign",
+ "absent": true
+ },
+ {
+ "concept": "the old flow was bad",
+ "absent": true
+ },
+ {
+ "concept": "users prefer the new design",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-06-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-06-summary.md
new file mode 100644
index 0000000..b551743
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-06-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-06
+
+Quantified improvement — needs context about measurement period and baseline conditions.
+
+## Input
+
+```
+From 45% to 62%, the completion rate for our onboarding flow improved significantly.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: observed_problem
+- **Actual Reasoning Modes**: identify_difference
+- **Response Duration**: 1ms
+
+### Technical Failures
+
+classification mismatch
+
+### Reasoning Quality Failures
+
+missing required concept(s): completion rate, 45%, 62%, onboarding
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-07-result.json b/evaluation-results/2026-08-01T06-01-56/diag-07-result.json
new file mode 100644
index 0000000..e902e9c
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-07-result.json
@@ -0,0 +1,56 @@
+{
+ "id": "diag-07",
+ "description": "Single reported claim — needs validation, not acceptance as fact.",
+ "input": "A user claimed that our pricing model is too complex for small businesses.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "pricing complexity",
+ "found": false
+ },
+ {
+ "concept": "small business",
+ "found": false
+ },
+ {
+ "concept": "user claim",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "the pricing is actually complex",
+ "absent": true
+ },
+ {
+ "concept": "other small businesses agree",
+ "absent": true
+ },
+ {
+ "concept": "we should simplify pricing",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-07-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-07-summary.md
new file mode 100644
index 0000000..4a8a7d7
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-07-summary.md
@@ -0,0 +1,21 @@
+# Diagnostic Case: diag-07
+
+Single reported claim — needs validation, not acceptance as fact.
+
+## Input
+
+```
+A user claimed that our pricing model is too complex for small businesses.
+```
+
+## Result
+
+- **Technical**: ✅ PASS (1/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: reported_claim
+- **Actual Reasoning Modes**: identify_difference, validate_claim
+- **Response Duration**: 1ms
+
+### Reasoning Quality Failures
+
+missing required concept(s): pricing complexity, small business, user claim
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-08-result.json b/evaluation-results/2026-08-01T06-01-56/diag-08-result.json
new file mode 100644
index 0000000..bdbef7d
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-08-result.json
@@ -0,0 +1,56 @@
+{
+ "id": "diag-08",
+ "description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode.",
+ "input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "ambiguous_statement",
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "philosophical",
+ "found": false
+ },
+ {
+ "concept": "ambiguous",
+ "found": true
+ },
+ {
+ "concept": "meaning clarification",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "the phrase was wrong",
+ "absent": true
+ },
+ {
+ "concept": "the colleague is hostile",
+ "absent": true
+ },
+ {
+ "concept": "we should avoid philosophical language",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-08-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-08-summary.md
new file mode 100644
index 0000000..b5cd5a8
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-08-summary.md
@@ -0,0 +1,21 @@
+# Diagnostic Case: diag-08
+
+Meta-test — self-referential ambiguous statement. Should trigger clarification mode.
+
+## Input
+
+```
+I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?
+```
+
+## Result
+
+- **Technical**: ✅ PASS (1/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: ambiguous_statement
+- **Actual Reasoning Modes**: identify_difference, clarify_meaning
+- **Response Duration**: 1ms
+
+### Reasoning Quality Failures
+
+missing required concept(s): philosophical, meaning clarification
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-09-result.json b/evaluation-results/2026-08-01T06-01-56/diag-09-result.json
new file mode 100644
index 0000000..367933f
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-09-result.json
@@ -0,0 +1,60 @@
+{
+ "id": "diag-09",
+ "description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.",
+ "input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "deployment",
+ "found": false
+ },
+ {
+ "concept": "complaint volume increase",
+ "found": false
+ },
+ {
+ "concept": "tripled",
+ "found": false
+ },
+ {
+ "concept": "47 cases",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "the deployment caused the complaints",
+ "absent": true
+ },
+ {
+ "concept": "the bug report was insufficient",
+ "absent": true
+ },
+ {
+ "concept": "rollback is needed",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-09-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-09-summary.md
new file mode 100644
index 0000000..a9a517f
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-09-summary.md
@@ -0,0 +1,21 @@
+# Diagnostic Case: diag-09
+
+Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.
+
+## Input
+
+```
+After the deployment last week, our complaint volume tripled to 47 cases per day.
+```
+
+## Result
+
+- **Technical**: ✅ PASS (1/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: causal_claim
+- **Actual Reasoning Modes**: establish_baseline, identify_difference
+- **Response Duration**: 1ms
+
+### Reasoning Quality Failures
+
+missing required concept(s): deployment, complaint volume increase, tripled, 47 cases
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-10-result.json b/evaluation-results/2026-08-01T06-01-56/diag-10-result.json
new file mode 100644
index 0000000..5e1fcab
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-10-result.json
@@ -0,0 +1,56 @@
+{
+ "id": "diag-10",
+ "description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims.",
+ "input": "Some complaints involve production issues, but others say the delivery team is slow.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "production issues",
+ "found": false
+ },
+ {
+ "concept": "delivery speed",
+ "found": false
+ },
+ {
+ "concept": "complaint types",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "production is worse than delivery",
+ "absent": true
+ },
+ {
+ "concept": "the delivery team needs training",
+ "absent": true
+ },
+ {
+ "concept": "both teams are underperforming equally",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-01-56/diag-10-summary.md b/evaluation-results/2026-08-01T06-01-56/diag-10-summary.md
new file mode 100644
index 0000000..ad5a73c
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/diag-10-summary.md
@@ -0,0 +1,21 @@
+# Diagnostic Case: diag-10
+
+Paired with diag-01 — distinguishes subset complaints from aggregate claims.
+
+## Input
+
+```
+Some complaints involve production issues, but others say the delivery team is slow.
+```
+
+## Result
+
+- **Technical**: ✅ PASS (1/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: observed_problem
+- **Actual Reasoning Modes**: establish_baseline, identify_difference
+- **Response Duration**: 1ms
+
+### Reasoning Quality Failures
+
+missing required concept(s): production issues, delivery speed, complaint types
diff --git a/evaluation-results/2026-08-01T06-01-56/summary.json b/evaluation-results/2026-08-01T06-01-56/summary.json
new file mode 100644
index 0000000..d0bde97
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-01-56/summary.json
@@ -0,0 +1,585 @@
+{
+ "timestamp": "2026-08-01T06:01:56.070Z",
+ "provider": "mock",
+ "promptVersion": "v0.2",
+ "casesRun": 10,
+ "summary": {
+ "technical": {
+ "schemaValidityRate": "100.0%",
+ "classificationMatchRate": "80.0%",
+ "nextQuestionPresentRate": "100.0%",
+ "passRate": "80.0%"
+ },
+ "reasoningQuality": {
+ "requiredConceptMatchRate": "0.0%",
+ "unsupportedInferenceFailures": "0",
+ "passRate": "0.0%"
+ },
+ "combinedPassRate": "0.0%",
+ "averageResponseDurationMs": "2"
+ },
+ "testCaseResults": [
+ {
+ "id": "diag-01",
+ "input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
+ "responseDurationMs": 6,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "complaints",
+ "found": true
+ },
+ {
+ "concept": "warehouse",
+ "found": false
+ },
+ {
+ "concept": "baseline comparison",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "quality issue",
+ "absent": true
+ },
+ {
+ "concept": "staff turnover",
+ "absent": true
+ },
+ {
+ "concept": "training gap",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-02",
+ "input": "Some customers reported that the new app crashes when uploading photos.",
+ "responseDurationMs": 2,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "app crashes",
+ "found": false
+ },
+ {
+ "concept": "photo upload",
+ "found": false
+ },
+ {
+ "concept": "some customers",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "all users affected",
+ "absent": true
+ },
+ {
+ "concept": "server-side bug",
+ "absent": true
+ },
+ {
+ "concept": "Android only",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-03",
+ "input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": false,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "sales decline",
+ "found": false
+ },
+ {
+ "concept": "price increase",
+ "found": false
+ },
+ {
+ "concept": "revenue increase",
+ "found": false
+ },
+ {
+ "concept": "CFO report",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "price was set too high",
+ "absent": true
+ },
+ {
+ "concept": "competitors gained market share",
+ "absent": true
+ },
+ {
+ "concept": "revenue data is wrong",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-04",
+ "input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "marketplace app",
+ "found": false
+ },
+ {
+ "concept": "Southeast Asia",
+ "found": false
+ },
+ {
+ "concept": "competitor gap",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "this will definitely succeed",
+ "absent": true
+ },
+ {
+ "concept": "we have the resources",
+ "absent": true
+ },
+ {
+ "concept": "competitors are struggling",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-05",
+ "input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "supplier change",
+ "found": false
+ },
+ {
+ "concept": "three months ago",
+ "found": false
+ },
+ {
+ "concept": "same defect rate",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "new supplier is worse",
+ "absent": true
+ },
+ {
+ "concept": "old supplier was better",
+ "absent": true
+ },
+ {
+ "concept": "quality process is broken",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-06",
+ "input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": false,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "completion rate",
+ "found": false
+ },
+ {
+ "concept": "45%",
+ "found": false
+ },
+ {
+ "concept": "62%",
+ "found": false
+ },
+ {
+ "concept": "onboarding",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "all improvements are due to the redesign",
+ "absent": true
+ },
+ {
+ "concept": "the old flow was bad",
+ "absent": true
+ },
+ {
+ "concept": "users prefer the new design",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-07",
+ "input": "A user claimed that our pricing model is too complex for small businesses.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "pricing complexity",
+ "found": false
+ },
+ {
+ "concept": "small business",
+ "found": false
+ },
+ {
+ "concept": "user claim",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "the pricing is actually complex",
+ "absent": true
+ },
+ {
+ "concept": "other small businesses agree",
+ "absent": true
+ },
+ {
+ "concept": "we should simplify pricing",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-08",
+ "input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "ambiguous_statement",
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "philosophical",
+ "found": false
+ },
+ {
+ "concept": "ambiguous",
+ "found": true
+ },
+ {
+ "concept": "meaning clarification",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "the phrase was wrong",
+ "absent": true
+ },
+ {
+ "concept": "the colleague is hostile",
+ "absent": true
+ },
+ {
+ "concept": "we should avoid philosophical language",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-09",
+ "input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "deployment",
+ "found": false
+ },
+ {
+ "concept": "complaint volume increase",
+ "found": false
+ },
+ {
+ "concept": "tripled",
+ "found": false
+ },
+ {
+ "concept": "47 cases",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "the deployment caused the complaints",
+ "absent": true
+ },
+ {
+ "concept": "the bug report was insufficient",
+ "absent": true
+ },
+ {
+ "concept": "rollback is needed",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-10",
+ "input": "Some complaints involve production issues, but others say the delivery team is slow.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "production issues",
+ "found": false
+ },
+ {
+ "concept": "delivery speed",
+ "found": false
+ },
+ {
+ "concept": "complaint types",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "production is worse than delivery",
+ "absent": true
+ },
+ {
+ "concept": "the delivery team needs training",
+ "absent": true
+ },
+ {
+ "concept": "both teams are underperforming equally",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-01-result.json b/evaluation-results/2026-08-01T06-07-42/diag-01-result.json
new file mode 100644
index 0000000..5e47ef7
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-01-result.json
@@ -0,0 +1,30 @@
+{
+ "id": "diag-01",
+ "description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.",
+ "input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
+ "responseDurationMs": 1785564462515,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-01-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-01-summary.md
new file mode 100644
index 0000000..1570f54
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-01-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-01
+
+Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.
+
+## Input
+
+```
+We've seen a spike in complaints from our warehouse team this month compared to last month.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 1785564462515ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-02-result.json b/evaluation-results/2026-08-01T06-07-42/diag-02-result.json
new file mode 100644
index 0000000..1e9e5ed
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-02-result.json
@@ -0,0 +1,30 @@
+{
+ "id": "diag-02",
+ "description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.",
+ "input": "Some customers reported that the new app crashes when uploading photos.",
+ "responseDurationMs": 1785564462519,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-02-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-02-summary.md
new file mode 100644
index 0000000..0518536
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-02-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-02
+
+Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.
+
+## Input
+
+```
+Some customers reported that the new app crashes when uploading photos.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 1785564462519ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-03-result.json b/evaluation-results/2026-08-01T06-07-42/diag-03-result.json
new file mode 100644
index 0000000..4d30197
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-03-result.json
@@ -0,0 +1,30 @@
+{
+ "id": "diag-03",
+ "description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.",
+ "input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
+ "responseDurationMs": 1785564462519,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-03-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-03-summary.md
new file mode 100644
index 0000000..7180883
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-03-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-03
+
+Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.
+
+## Input
+
+```
+Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 1785564462519ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-04-result.json b/evaluation-results/2026-08-01T06-07-42/diag-04-result.json
new file mode 100644
index 0000000..d62c283
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-04-result.json
@@ -0,0 +1,30 @@
+{
+ "id": "diag-04",
+ "description": "Decision request — forward-looking, needs missing info identification.",
+ "input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
+ "responseDurationMs": 1785564462520,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-04-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-04-summary.md
new file mode 100644
index 0000000..7229c5d
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-04-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-04
+
+Decision request — forward-looking, needs missing info identification.
+
+## Input
+
+```
+We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 1785564462520ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-05-result.json b/evaluation-results/2026-08-01T06-07-42/diag-05-result.json
new file mode 100644
index 0000000..f957f54
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-05-result.json
@@ -0,0 +1,30 @@
+{
+ "id": "diag-05",
+ "description": "Unexpected continuity — changed context but no outcome change.",
+ "input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
+ "responseDurationMs": 1785564462520,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-05-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-05-summary.md
new file mode 100644
index 0000000..0389218
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-05-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-05
+
+Unexpected continuity — changed context but no outcome change.
+
+## Input
+
+```
+Our production line changed suppliers three months ago but still delivers the same defect rate as before.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 1785564462520ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-06-result.json b/evaluation-results/2026-08-01T06-07-42/diag-06-result.json
new file mode 100644
index 0000000..f4255c3
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-06-result.json
@@ -0,0 +1,30 @@
+{
+ "id": "diag-06",
+ "description": "Quantified improvement — needs context about measurement period and baseline conditions.",
+ "input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
+ "responseDurationMs": 1785564462521,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-06-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-06-summary.md
new file mode 100644
index 0000000..02da337
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-06-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-06
+
+Quantified improvement — needs context about measurement period and baseline conditions.
+
+## Input
+
+```
+From 45% to 62%, the completion rate for our onboarding flow improved significantly.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 1785564462521ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-07-result.json b/evaluation-results/2026-08-01T06-07-42/diag-07-result.json
new file mode 100644
index 0000000..fcdb417
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-07-result.json
@@ -0,0 +1,30 @@
+{
+ "id": "diag-07",
+ "description": "Single reported claim — needs validation, not acceptance as fact.",
+ "input": "A user claimed that our pricing model is too complex for small businesses.",
+ "responseDurationMs": 1785564462521,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-07-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-07-summary.md
new file mode 100644
index 0000000..54e004f
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-07-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-07
+
+Single reported claim — needs validation, not acceptance as fact.
+
+## Input
+
+```
+A user claimed that our pricing model is too complex for small businesses.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 1785564462521ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-08-result.json b/evaluation-results/2026-08-01T06-07-42/diag-08-result.json
new file mode 100644
index 0000000..e07f754
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-08-result.json
@@ -0,0 +1,30 @@
+{
+ "id": "diag-08",
+ "description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode.",
+ "input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
+ "responseDurationMs": 1785564462521,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-08-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-08-summary.md
new file mode 100644
index 0000000..ff01aa7
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-08-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-08
+
+Meta-test — self-referential ambiguous statement. Should trigger clarification mode.
+
+## Input
+
+```
+I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 1785564462521ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-09-result.json b/evaluation-results/2026-08-01T06-07-42/diag-09-result.json
new file mode 100644
index 0000000..75a0fce
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-09-result.json
@@ -0,0 +1,30 @@
+{
+ "id": "diag-09",
+ "description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.",
+ "input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
+ "responseDurationMs": 1785564462521,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-09-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-09-summary.md
new file mode 100644
index 0000000..7dafaba
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-09-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-09
+
+Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.
+
+## Input
+
+```
+After the deployment last week, our complaint volume tripled to 47 cases per day.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 1785564462521ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-10-result.json b/evaluation-results/2026-08-01T06-07-42/diag-10-result.json
new file mode 100644
index 0000000..7f1de1f
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-10-result.json
@@ -0,0 +1,30 @@
+{
+ "id": "diag-10",
+ "description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims.",
+ "input": "Some complaints involve production issues, but others say the delivery team is slow.",
+ "responseDurationMs": 1785564462522,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-07-42/diag-10-summary.md b/evaluation-results/2026-08-01T06-07-42/diag-10-summary.md
new file mode 100644
index 0000000..9e20b0b
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/diag-10-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-10
+
+Paired with diag-01 — distinguishes subset complaints from aggregate claims.
+
+## Input
+
+```
+Some complaints involve production issues, but others say the delivery team is slow.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 1785564462522ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-07-42/summary.json b/evaluation-results/2026-08-01T06-07-42/summary.json
new file mode 100644
index 0000000..37bdef7
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-07-42/summary.json
@@ -0,0 +1,313 @@
+{
+ "timestamp": "2026-08-01T06:07:42.532Z",
+ "provider": "ollama-real",
+ "promptVersion": "v0.2",
+ "casesRun": 10,
+ "summary": {
+ "technical": {
+ "schemaValidityRate": "0.0%",
+ "classificationMatchRate": "0.0%",
+ "nextQuestionPresentRate": "0.0%",
+ "passRate": "0.0%"
+ },
+ "reasoningQuality": {
+ "requiredConceptMatchRate": "100.0%",
+ "unsupportedInferenceFailures": "0",
+ "passRate": "0.0%"
+ },
+ "combinedPassRate": "0.0%",
+ "averageResponseDurationMs": "1785564462520"
+ },
+ "testCaseResults": [
+ {
+ "id": "diag-01",
+ "input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
+ "responseDurationMs": 1785564462515,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-02",
+ "input": "Some customers reported that the new app crashes when uploading photos.",
+ "responseDurationMs": 1785564462519,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-03",
+ "input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
+ "responseDurationMs": 1785564462519,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-04",
+ "input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
+ "responseDurationMs": 1785564462520,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-05",
+ "input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
+ "responseDurationMs": 1785564462520,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-06",
+ "input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
+ "responseDurationMs": 1785564462521,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-07",
+ "input": "A user claimed that our pricing model is too complex for small businesses.",
+ "responseDurationMs": 1785564462521,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-08",
+ "input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
+ "responseDurationMs": 1785564462521,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-09",
+ "input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
+ "responseDurationMs": 1785564462521,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-10",
+ "input": "Some complaints involve production issues, but others say the delivery team is slow.",
+ "responseDurationMs": 1785564462522,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Invalid server configuration",
+ "Invalid server configuration"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-01-result.json b/evaluation-results/2026-08-01T06-11-41/diag-01-result.json
new file mode 100644
index 0000000..0ceb523
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-01-result.json
@@ -0,0 +1,33 @@
+{
+ "id": "diag-01",
+ "description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.",
+ "input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
+ "responseDurationMs": 19459,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "rawOutput": "{\"status\":\"received\",\"message\":\"Please provide a specific request or data to process.\"}",
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "inputClassification: Required",
+ "reconstruction: Required",
+ "evidence: Required",
+ "nextQuestion: Required"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-01-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-01-summary.md
new file mode 100644
index 0000000..c68eeac
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-01-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-01
+
+Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.
+
+## Input
+
+```
+We've seen a spike in complaints from our warehouse team this month compared to last month.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 19459ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-02-result.json b/evaluation-results/2026-08-01T06-11-41/diag-02-result.json
new file mode 100644
index 0000000..0120dce
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-02-result.json
@@ -0,0 +1,30 @@
+{
+ "id": "diag-02",
+ "description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.",
+ "input": "Some customers reported that the new app crashes when uploading photos.",
+ "responseDurationMs": 14693,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download",
+ "Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-02-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-02-summary.md
new file mode 100644
index 0000000..706b324
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-02-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-02
+
+Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.
+
+## Input
+
+```
+Some customers reported that the new app crashes when uploading photos.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 14693ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-03-result.json b/evaluation-results/2026-08-01T06-11-41/diag-03-result.json
new file mode 100644
index 0000000..3aa6882
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-03-result.json
@@ -0,0 +1,33 @@
+{
+ "id": "diag-03",
+ "description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.",
+ "input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
+ "responseDurationMs": 13516,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "rawOutput": "{\"status\":\"success\",\"message\":\"Input received and processed.\"}",
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "inputClassification: Required",
+ "reconstruction: Required",
+ "evidence: Required",
+ "nextQuestion: Required"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-03-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-03-summary.md
new file mode 100644
index 0000000..0a4c7f3
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-03-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-03
+
+Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.
+
+## Input
+
+```
+Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 13516ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-04-result.json b/evaluation-results/2026-08-01T06-11-41/diag-04-result.json
new file mode 100644
index 0000000..881b637
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-04-result.json
@@ -0,0 +1,33 @@
+{
+ "id": "diag-04",
+ "description": "Decision request — forward-looking, needs missing info identification.",
+ "input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
+ "responseDurationMs": 48851,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "rawOutput": "{\"state\":\"pending\"}",
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "inputClassification: Required",
+ "reconstruction: Required",
+ "evidence: Required",
+ "nextQuestion: Required"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-04-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-04-summary.md
new file mode 100644
index 0000000..dcf3fe9
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-04-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-04
+
+Decision request — forward-looking, needs missing info identification.
+
+## Input
+
+```
+We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 48851ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-05-result.json b/evaluation-results/2026-08-01T06-11-41/diag-05-result.json
new file mode 100644
index 0000000..2f5064d
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-05-result.json
@@ -0,0 +1,33 @@
+{
+ "id": "diag-05",
+ "description": "Unexpected continuity — changed context but no outcome change.",
+ "input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
+ "responseDurationMs": 16508,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "rawOutput": "{}",
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "inputClassification: Required",
+ "reconstruction: Required",
+ "evidence: Required",
+ "nextQuestion: Required"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-05-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-05-summary.md
new file mode 100644
index 0000000..4a87c63
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-05-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-05
+
+Unexpected continuity — changed context but no outcome change.
+
+## Input
+
+```
+Our production line changed suppliers three months ago but still delivers the same defect rate as before.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 16508ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-06-result.json b/evaluation-results/2026-08-01T06-11-41/diag-06-result.json
new file mode 100644
index 0000000..923a8c9
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-06-result.json
@@ -0,0 +1,33 @@
+{
+ "id": "diag-06",
+ "description": "Quantified improvement — needs context about measurement period and baseline conditions.",
+ "input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
+ "responseDurationMs": 15608,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "rawOutput": "{\"status\":\"ok\"}",
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "inputClassification: Required",
+ "reconstruction: Required",
+ "evidence: Required",
+ "nextQuestion: Required"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-06-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-06-summary.md
new file mode 100644
index 0000000..0a02208
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-06-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-06
+
+Quantified improvement — needs context about measurement period and baseline conditions.
+
+## Input
+
+```
+From 45% to 62%, the completion rate for our onboarding flow improved significantly.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 15608ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-07-result.json b/evaluation-results/2026-08-01T06-11-41/diag-07-result.json
new file mode 100644
index 0000000..2476ec3
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-07-result.json
@@ -0,0 +1,33 @@
+{
+ "id": "diag-07",
+ "description": "Single reported claim — needs validation, not acceptance as fact.",
+ "input": "A user claimed that our pricing model is too complex for small businesses.",
+ "responseDurationMs": 15750,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "rawOutput": "{\"status\":\"success\",\"message\":\"Input received\"}",
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "inputClassification: Required",
+ "reconstruction: Required",
+ "evidence: Required",
+ "nextQuestion: Required"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-07-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-07-summary.md
new file mode 100644
index 0000000..cbd3515
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-07-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-07
+
+Single reported claim — needs validation, not acceptance as fact.
+
+## Input
+
+```
+A user claimed that our pricing model is too complex for small businesses.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 15750ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-08-result.json b/evaluation-results/2026-08-01T06-11-41/diag-08-result.json
new file mode 100644
index 0000000..9ce18ae
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-08-result.json
@@ -0,0 +1,33 @@
+{
+ "id": "diag-08",
+ "description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode.",
+ "input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
+ "responseDurationMs": 15383,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "rawOutput": "{\"status\":\"success\",\"message\":\"JSON object returned as requested\"}",
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "inputClassification: Required",
+ "reconstruction: Required",
+ "evidence: Required",
+ "nextQuestion: Required"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-08-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-08-summary.md
new file mode 100644
index 0000000..fed9f98
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-08-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-08
+
+Meta-test — self-referential ambiguous statement. Should trigger clarification mode.
+
+## Input
+
+```
+I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 15383ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-09-result.json b/evaluation-results/2026-08-01T06-11-41/diag-09-result.json
new file mode 100644
index 0000000..f80452c
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-09-result.json
@@ -0,0 +1,33 @@
+{
+ "id": "diag-09",
+ "description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.",
+ "input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
+ "responseDurationMs": 15643,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "rawOutput": "{\"status\":\"ok\"}",
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "inputClassification: Required",
+ "reconstruction: Required",
+ "evidence: Required",
+ "nextQuestion: Required"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-09-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-09-summary.md
new file mode 100644
index 0000000..fb68c09
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-09-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-09
+
+Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.
+
+## Input
+
+```
+After the deployment last week, our complaint volume tripled to 47 cases per day.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 15643ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-10-result.json b/evaluation-results/2026-08-01T06-11-41/diag-10-result.json
new file mode 100644
index 0000000..9f59f16
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-10-result.json
@@ -0,0 +1,33 @@
+{
+ "id": "diag-10",
+ "description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims.",
+ "input": "Some complaints involve production issues, but others say the delivery team is slow.",
+ "responseDurationMs": 15401,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "rawOutput": "{}",
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "inputClassification: Required",
+ "reconstruction: Required",
+ "evidence: Required",
+ "nextQuestion: Required"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+}
\ No newline at end of file
diff --git a/evaluation-results/2026-08-01T06-11-41/diag-10-summary.md b/evaluation-results/2026-08-01T06-11-41/diag-10-summary.md
new file mode 100644
index 0000000..3067f33
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/diag-10-summary.md
@@ -0,0 +1,25 @@
+# Diagnostic Case: diag-10
+
+Paired with diag-01 — distinguishes subset complaints from aggregate claims.
+
+## Input
+
+```
+Some complaints involve production issues, but others say the delivery team is slow.
+```
+
+## Result
+
+- **Technical**: ❌ FAIL (0/4 sub-checks pass)
+- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
+- **Actual Primary Type**: N/A
+- **Actual Reasoning Modes**: N/A
+- **Response Duration**: 15401ms
+
+### Technical Failures
+
+schema invalid, classification mismatch, no next question
+
+### Reasoning Quality Failures
+
+
diff --git a/evaluation-results/2026-08-01T06-11-41/summary.json b/evaluation-results/2026-08-01T06-11-41/summary.json
new file mode 100644
index 0000000..3e65eac
--- /dev/null
+++ b/evaluation-results/2026-08-01T06-11-41/summary.json
@@ -0,0 +1,331 @@
+{
+ "timestamp": "2026-08-01T06:11:41.538Z",
+ "provider": "ollama-real",
+ "promptVersion": "v0.2",
+ "casesRun": 10,
+ "summary": {
+ "technical": {
+ "schemaValidityRate": "0.0%",
+ "classificationMatchRate": "0.0%",
+ "nextQuestionPresentRate": "0.0%",
+ "passRate": "0.0%"
+ },
+ "reasoningQuality": {
+ "requiredConceptMatchRate": "100.0%",
+ "unsupportedInferenceFailures": "0",
+ "passRate": "0.0%"
+ },
+ "combinedPassRate": "0.0%",
+ "averageResponseDurationMs": "19081"
+ },
+ "testCaseResults": [
+ {
+ "id": "diag-01",
+ "input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
+ "responseDurationMs": 19459,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "inputClassification: Required",
+ "reconstruction: Required",
+ "evidence: Required",
+ "nextQuestion: Required"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-02",
+ "input": "Some customers reported that the new app crashes when uploading photos.",
+ "responseDurationMs": 14693,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download",
+ "Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-03",
+ "input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
+ "responseDurationMs": 13516,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "inputClassification: Required",
+ "reconstruction: Required",
+ "evidence: Required",
+ "nextQuestion: Required"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-04",
+ "input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
+ "responseDurationMs": 48851,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "inputClassification: Required",
+ "reconstruction: Required",
+ "evidence: Required",
+ "nextQuestion: Required"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-05",
+ "input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
+ "responseDurationMs": 16508,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "inputClassification: Required",
+ "reconstruction: Required",
+ "evidence: Required",
+ "nextQuestion: Required"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-06",
+ "input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
+ "responseDurationMs": 15608,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "inputClassification: Required",
+ "reconstruction: Required",
+ "evidence: Required",
+ "nextQuestion: Required"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-07",
+ "input": "A user claimed that our pricing model is too complex for small businesses.",
+ "responseDurationMs": 15750,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "inputClassification: Required",
+ "reconstruction: Required",
+ "evidence: Required",
+ "nextQuestion: Required"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-08",
+ "input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
+ "responseDurationMs": 15383,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "inputClassification: Required",
+ "reconstruction: Required",
+ "evidence: Required",
+ "nextQuestion: Required"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-09",
+ "input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
+ "responseDurationMs": 15643,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "inputClassification: Required",
+ "reconstruction: Required",
+ "evidence: Required",
+ "nextQuestion: Required"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "diag-10",
+ "input": "Some complaints involve production issues, but others say the delivery team is slow.",
+ "responseDurationMs": 15401,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [],
+ "technical": {
+ "schemaValid": false,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": false,
+ "pass": false,
+ "errors": [
+ "inputClassification: Required",
+ "reconstruction: Required",
+ "evidence: Required",
+ "nextQuestion: Required"
+ ]
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": true,
+ "details": []
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/evaluation-results/latest-manifest.json b/evaluation-results/latest-manifest.json
new file mode 100644
index 0000000..18f27f5
--- /dev/null
+++ b/evaluation-results/latest-manifest.json
@@ -0,0 +1,4 @@
+{
+ "latestRun": "2026-08-01T06-11-41",
+ "caseCount": 10
+}
\ No newline at end of file
diff --git a/lib/analysis.js b/lib/analysis.js
new file mode 100644
index 0000000..8741cf1
--- /dev/null
+++ b/lib/analysis.js
@@ -0,0 +1,182 @@
+/**
+ * Core analysis pipeline — shared by API routes and evaluation harness.
+ * Calls the provider, parses output, validates against Zod schemas (v0.2 first, v0.1 fallback).
+ */
+
+import { getConfig } from "../lib/config.js";
+import { getProvider } from "../lib/llm/provider.js";
+import { buildPrompt, PROMPT_VERSIONS } from "../lib/reconstruction/prompt.js";
+import {
+ reconstructionV2Schema,
+ reconstructionSchema as reconstructionV1Schema,
+} from "../lib/reconstruction/schema.js";
+
+const MAX_SCENARIO_LENGTH = 10000;
+const DEFAULT_PROMPT_VERSION = "v0.2";
+
+/**
+ * Analyse a scenario string through the full pipeline.
+ * @param {string} scenario - The scenario text to analyse
+ * @param {object} [opts]
+ * @param {"v0.1" | "v0.2"} [opts.promptVersion="v0.2"] - Prompt version to use
+ * @returns {Promise} Analysis result with diagnostics
+ */
+export async function analyseScenario(scenario, opts = {}) {
+ const startTime = Date.now();
+
+ // ── Input validation ───────────────────────────────
+ if (typeof scenario !== "string") {
+ return buildErrorResponse("Input must be a string", startTime);
+ }
+
+ const trimmed = scenario.trim();
+ if (trimmed.length === 0) {
+ return buildErrorResponse("Scenario cannot be empty", startTime);
+ }
+ if (trimmed.length > MAX_SCENARIO_LENGTH) {
+ return buildErrorResponse(`Scenario must be under ${MAX_SCENARIO_LENGTH} characters`, startTime);
+ }
+
+ // ── Configuration check ────────────────────────────
+ const configResult = getConfig();
+ if (!configResult.ok) {
+ return buildErrorResponse("Invalid server configuration", startTime, "500");
+ }
+
+ const { OLLAMA_BASE_URL: _ignored, OLLAMA_MODEL } = configResult.config;
+ const promptVersion = opts.promptVersion || DEFAULT_PROMPT_VERSION;
+
+ // ── Build prompt ───────────────────────────────────
+ let promptObj;
+ try {
+ promptObj = await buildPrompt(trimmed, promptVersion);
+ } catch (e) {
+ return buildErrorResponse(`Failed to build prompt: ${e.message}`, startTime);
+ }
+
+ // ── Call provider ──────────────────────────────────
+ const provider = getProvider();
+ let rawResponse;
+ try {
+ rawResponse = await provider.generateReconstruction(promptObj.prompt, OLLAMA_MODEL);
+ } catch (e) {
+ return buildErrorResponse(
+ e.message || "Provider error during analysis",
+ Date.now() - startTime
+ );
+ }
+
+ const duration = Date.now() - startTime;
+
+ // Try to capture raw response for diagnostics
+ let rawResponseStr;
+ try {
+ rawResponseStr = JSON.stringify(rawResponse);
+ } catch {
+ rawResponseStr = String(rawResponse).slice(0, 2000);
+ }
+
+ // ── Validate against v0.2 schema (preferred) ──────
+ const resultV2 = tryValidateAgainstSchema(rawResponse, reconstructionV2Schema);
+ if (resultV2.valid) {
+ return buildSuccessResultV2(resultV2.data, OLLAMA_MODEL, duration, promptVersion);
+ }
+
+ // ── Fallback to v0.1 schema ────────────────────────
+ const resultV1 = tryValidateAgainstSchema(rawResponse, reconstructionV1Schema);
+ if (resultV1.valid) {
+ return buildSuccessResultV1(resultV1.data, OLLAMA_MODEL, duration, promptVersion);
+ }
+
+ // ── Neither schema matched — partial failure ───────
+ return buildPartialResult(
+ rawResponseStr?.slice(0, 2000),
+ resultV2.error ?? resultV1.error,
+ OLLAMA_MODEL,
+ duration,
+ promptVersion
+ );
+}
+
+/** Attempt validation against a Zod schema */
+function tryValidateAgainstSchema(data, schema) {
+ if (!schema.safeParse) {
+ return { valid: false, error: new Error("Schema does not support safeParse") };
+ }
+ const result = schema.safeParse(data);
+ return result.success ? { valid: true, data: result.data } : { valid: false, error: result.error };
+}
+
+// ── Result builders ──────────────────────────────────
+
+function buildErrorResponse(message, elapsed, statusCode = 500) {
+ return {
+ success: false,
+ error: message,
+ modelName: null,
+ responseDurationMs: elapsed,
+ validationStatus: "invalid",
+ rawResponse: null,
+ promptVersion: null,
+ statusCode,
+ };
+}
+
+function buildSuccessResultV2(data, model, duration, version) {
+ return {
+ success: true,
+ validationStatus: "valid",
+ modelName: model,
+ responseDurationMs: duration,
+ rawResponse: JSON.stringify(data).slice(0, 3000),
+ promptVersion: version,
+ inputClassification: data.inputClassification,
+ reconstruction: data.reconstruction,
+ evidence: data.evidence,
+ nextQuestion: data.nextQuestion,
+ errors: undefined,
+ };
+}
+
+function buildSuccessResultV1(data, model, duration, version) {
+ return {
+ success: true,
+ validationStatus: "valid",
+ modelName: model,
+ responseDurationMs: duration,
+ rawResponse: JSON.stringify(data).slice(0, 3000),
+ promptVersion: version,
+ inputClassification: null,
+ reconstruction: data,
+ evidence: undefined,
+ nextQuestion: undefined,
+ errors: undefined,
+ };
+}
+
+function buildPartialResult(rawResp, error, model, duration, version) {
+ let errors = [];
+ if (error && typeof error.flatten === "function") {
+ errors = error.flatten().fieldErrors
+ ? Object.entries(error.flatten().fieldErrors).flatMap(([k, v]) => [`${k}: ${v.join(", ")}`])
+ : [String(error)];
+ } else if (error) {
+ errors = [String(error).slice(0, 500)];
+ }
+
+ return {
+ success: false,
+ validationStatus: "invalid",
+ modelName: model,
+ responseDurationMs: duration,
+ rawResponse: rawResp?.slice(0, 2000),
+ promptVersion: version,
+ inputClassification: null,
+ reconstruction: null,
+ evidence: undefined,
+ nextQuestion: undefined,
+ errors,
+ };
+}
+
+export { PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION };
diff --git a/lib/llm/provider.js b/lib/llm/provider.js
index 716a8cc..c41a313 100644
--- a/lib/llm/provider.js
+++ b/lib/llm/provider.js
@@ -93,11 +93,9 @@ async function detectChatSupport(baseUrl) {
class OllamaLlmProvider {
async generateReconstruction(scenario, modelName) {
- const { buildPrompt } = await import("@/lib/reconstruction/prompt");
-
- let rawPrompt = buildPrompt(scenario);
- // Stronger JSON hint since we can't use format:json on older Ollama
- const prompt = rawPrompt + `\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.`;
+ // scenario is ALREADY a fully-built prompt text (built by analyseScenario).
+ // Do NOT call buildPrompt() again — that would double-wrap the prompt.
+ const prompt = scenario;
const baseUrl = process.env.OLLAMA_BASE_URL;
if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");
diff --git a/lib/reconstruction/prompt.js b/lib/reconstruction/prompt.js
index 691700d..73581c8 100644
--- a/lib/reconstruction/prompt.js
+++ b/lib/reconstruction/prompt.js
@@ -1,5 +1,17 @@
-export function buildPrompt(scenario) {
- return `You are a neutral analyst performing an evidence-based reconstruction of the following scenario.
+import { promises as fs } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { dirname, join } from "node:path";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const PROMPTS_DIR = join(__dirname, "../../prompts");
+
+/** Available prompt versions */
+export const PROMPT_VERSIONS = ["v0.1", "v0.2"];
+
+/** Build a v0.1 (extraction-only) prompt inline for backward compatibility */
+function buildV1Prompt(scenario) {
+ return `You are a neutral analyst performing an evidence-based reconstruction of the following scenario.
Rules:
1. Do NOT invent facts. Only include information present in the scenario or clearly implied.
@@ -29,3 +41,34 @@ Return valid JSON matching this structure exactly:
Return ONLY the JSON object. No markdown, no explanation, no preamble.`;
}
+
+/** Load a versioned prompt from disk and substitute {{SCENARIO}} */
+async function buildV2Prompt(scenario) {
+ try {
+ const content = await fs.readFile(join(PROMPTS_DIR, "reconstruct-v0.2.md"), "utf-8");
+ return content.replace("{{SCENARIO}}", scenario);
+ } catch {
+ // Fall back to v0.1 prompt if v0.2 file is missing
+ return buildV1Prompt(scenario);
+ }
+}
+
+/**
+ * Build an analysis prompt for the given version.
+ * @param {"v0.1" | "v0.2"} [version="v0.2"]
+ * @returns {Promise<{prompt: string, version: string}>}
+ */
+export async function buildPrompt(scenario, version = "v0.2") {
+ let prompt;
+ switch (version) {
+ case "v0.1":
+ prompt = buildV1Prompt(scenario);
+ break;
+ default: // v0.2
+ prompt = await buildV2Prompt(scenario);
+ break;
+ }
+
+ const strongJsonHint = "\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.";
+ return { prompt: prompt + strongJsonHint, version };
+}
diff --git a/lib/reconstruction/schema.js b/lib/reconstruction/schema.js
index f94e831..c982f68 100644
--- a/lib/reconstruction/schema.js
+++ b/lib/reconstruction/schema.js
@@ -1,38 +1,51 @@
import { z } from "zod";
-const confidenceEnum = z.enum(["low", "medium", "high"]);
+// ──────────────────────────────────────────────
+// Shared enums (v0.1 & v0.2)
+// ──────────────────────────────────────────────
-const itemSchema = z.object({
+export const confidenceEnum = z.enum(["low", "medium", "high"]);
+const importanceEnum = z.enum(["incidental", "supporting", "important", "critical"]);
+const expectedInfoValueEnum = z.enum(["low", "medium", "high"]);
+
+// ──────────────────────────────────────────────
+// v0.1 — extraction-only schema (preserved)
+// ──────────────────────────────────────────────
+
+const confidenceEnumV1 = z.enum(["low", "medium", "high"]);
+
+const itemSchemaV1 = z.object({
id: z.string().min(1),
description: z.string().min(1),
- confidence: confidenceEnum,
+ confidence: confidenceEnumV1,
});
export const reconstructionSchema = z.object({
- observations: z.array(itemSchema),
+ observations: z.array(itemSchemaV1),
reportedClaims: z.array(
- itemSchema.extend({
+ itemSchemaV1.extend({
attributedTo: z.union([z.string().min(1), z.null()]).optional().nullable(),
})
),
- assumptions: z.array(itemSchema),
- entities: z.array(itemSchema),
+ assumptions: z.array(itemSchemaV1),
+ entities: z.array(itemSchemaV1),
transitions: z.array(
- itemSchema.extend({
+ itemSchemaV1.extend({
entity: z.string().min(1),
previousState: z.string().min(1),
currentState: z.string().min(1),
explanationStatus: z.string().min(1),
})
),
- expectedButMissing: z.array(itemSchema),
- presentButUnexpected: z.array(itemSchema),
- contradictions: z.array(itemSchema),
- openUncertainties: z.array(itemSchema),
+ expectedButMissing: z.array(itemSchemaV1),
+ presentButUnexpected: z.array(itemSchemaV1),
+ contradictions: z.array(itemSchemaV1),
+ openUncertainties: z.array(itemSchemaV1),
});
+// v0.1 analyse response (used internally)
export const analyseResponseSchema = z.object({
- reconstruction: reconstructionSchema,
+ reconstruction: z.union([reconstructionSchema, z.null()]),
modelName: z.string(),
responseDurationMs: z.number(),
validationStatus: z.enum(["valid", "partial", "invalid"]),
@@ -48,6 +61,133 @@ export const healthResponseSchema = z.object({
error: z.string().nullable(),
});
+// ──────────────────────────────────────────────
+// v0.2 — reasoning classification + reconstruction
+// ──────────────────────────────────────────────
+
+export const inputTypes = /** @type {z.ZodType} */ (
+ z.enum([
+ "observed_problem",
+ "unexplained_change",
+ "contradiction",
+ "decision_request",
+ "causal_claim",
+ "reported_claim",
+ "fault_report",
+ "ambiguous_statement",
+ "question",
+ "desired_outcome",
+ "insufficient_context",
+ "other",
+ ])
+);
+
+export const reasoningModes = /** @type {z.ZodType} */ (
+ z.enum([
+ "establish_baseline",
+ "identify_difference",
+ "reconstruct_transition",
+ "decompose_aggregate",
+ "validate_measurement",
+ "validate_claim",
+ "investigate_contradiction",
+ "clarify_meaning",
+ "decision_support",
+ "fault_investigation",
+ "identify_missing_information",
+ "test_possible_explanations",
+ "other",
+ ])
+);
+
+const evidenceRecordSchema = z.object({
+ id: z.string().min(1),
+ description: z.string().min(1),
+ evidenceType: z.enum(["direct_observation", "reported_statement", "interpretation", "assumption", "inferred_relationship"]),
+ source: z.string().optional(),
+ attribution: z.string().nullable().optional(),
+ confidence: confidenceEnum,
+ importance: importanceEnum,
+});
+
+const reconstructionSchemaV2 = z.object({
+ summary: z.string().min(1),
+ actors: z.array(itemSchemaV1),
+ systemsOrObjects: z.array(itemSchemaV1),
+ expectedStates: z.array(itemSchemaV1),
+ observedStates: z.array(itemSchemaV1),
+ differences: z.array(itemSchemaV1),
+ knownTransitions: z.array(
+ itemSchemaV1.extend({
+ entity: z.string().min(1),
+ previousState: z.string().min(1),
+ currentState: z.string().min(1),
+ explanationStatus: z.string().min(1),
+ })
+ ),
+ unexplainedTransitions: z.array(
+ itemSchemaV1.extend({
+ entity: z.string().min(1).optional(),
+ previousState: z.string().min(1).optional(),
+ currentState: z.string().min(1).optional(),
+ })
+ ),
+ contradictions: z.array(itemSchemaV1),
+ importantUnknowns: z.array(itemSchemaV1),
+ plausibleInterpretations: z.array(
+ z.object({
+ id: z.string().min(1),
+ description: z.string().min(1),
+ supportingEvidenceIds: z.array(z.string()),
+ assumptionsRequired: z.array(z.string()).optional().default([]),
+ confidence: confidenceEnum,
+ })
+ ),
+});
+
+const inputClassificationSchema = z.object({
+ primaryType: inputTypes,
+ secondaryTypes: z.array(inputTypes).optional().default([]),
+ reasoningModes: z.array(reasoningModes).optional().default([]),
+ classificationReason: z.string().min(1),
+ confidence: confidenceEnum,
+});
+
+const nextQuestionSchema = z.object({
+ id: z.string().min(1),
+ question: z.string().min(1),
+ targets: z.array(z.string()),
+ reason: z.string().min(1),
+ expectedInformationValue: expectedInfoValueEnum,
+ reasoningMode: reasoningModes.optional().default("other"),
+});
+
+// v0.2 complete analysis response (what the model produces)
+export const reconstructionV2Schema = z.object({
+ inputClassification: inputClassificationSchema,
+ reconstruction: reconstructionSchemaV2,
+ evidence: z.array(evidenceRecordSchema),
+ nextQuestion: nextQuestionSchema,
+});
+
+// Outer wrapper for API return (includes diagnostics + v0.2 data)
+export const analyseResponseV2Schema = z.object({
+ inputClassification: inputClassificationSchema.optional(),
+ reconstruction: reconstructionSchemaV2.optional().nullable(),
+ evidence: z.array(evidenceRecordSchema).optional(),
+ nextQuestion: nextQuestionSchema.optional(),
+ modelName: z.string(),
+ responseDurationMs: z.number(),
+ validationStatus: z.enum(["valid", "partial", "invalid"]),
+ rawResponse: z.string().optional(),
+ errors: z.array(z.string()).optional(),
+ promptVersion: z.string().optional(),
+});
+
+// ──────────────────────────────────────────────
+// Parsing helpers
+// ──────────────────────────────────────────────
+
export function parseReconstruction(raw) {
if (typeof raw === "string") {
try {
@@ -58,3 +198,14 @@ export function parseReconstruction(raw) {
}
return reconstructionSchema.parse(raw);
}
+
+export function parseReconstructionV2(raw) {
+ if (typeof raw === "string") {
+ try {
+ raw = JSON.parse(raw);
+ } catch {
+ throw new SyntaxError("Model response is not valid JSON");
+ }
+ }
+ return reconstructionV2Schema.parse(raw);
+}
diff --git a/prompts/comparison-findings.md b/prompts/comparison-findings.md
new file mode 100644
index 0000000..70b65a0
--- /dev/null
+++ b/prompts/comparison-findings.md
@@ -0,0 +1,67 @@
+# v0.1 vs v0.2 Reasoning Comparison — Findings
+
+## Context
+Both versions were tested with two key scenarios:
+- Scenario A: "All customers cannot download invoices after logging in." (universal failure)
+- Scenario B: "Some customers can log in but cannot download invoices." (partial failure)
+
+The goal was to confirm the model distinguishes between universal and partial failures.
+
+## Results — v0.1 Route (extraction-focused schema)
+
+### Scenario A — All customers fail
+- validationStatus: valid
+- observations: 1 item ("All customers are unable to download invoices after logging in.")
+- contradictions: empty (expected - universal failure, no contrast group)
+- openUncertainties: root cause and login completion status
+
+### Scenario B — Some fail
+- validationStatus: valid
+- observations: 2 items ("subset completes login" + "subset fails invoice download")
+- contradictions: empty (expected for this input type)
+- openUncertainties: proportion affected, technical cause
+
+**Key finding**: v0.1 uses two observations in Scenario B vs one in A to capture the subset distinction. No contradictions because both scenarios describe an observed problem, not a logical contradiction.
+
+## Results — v0.2 Route (reasoning classification schema)
+
+### Scenario A — All customers fail
+- validationStatus: valid
+- primaryType: observed_problem + fault_report (secondary)
+- differences: empty (expected - universal failure has no contrast group)
+- importantUnknowns: error message, recent changes to services
+- reasoningModes: identify_difference, fault_investigation, identify_missing_information
+
+### Scenario B — Some fail
+- validationStatus: valid
+- primaryType: observed_problem + fault_report (secondary)
+- differences (1): "The failure is limited to some customers, implying a difference between affected and unaffected user accounts"
+- importantUnknowns: what distinguishes affected from unaffected accounts
+- reasoningModes: identify_difference, fault_investigation, identify_missing_information
+
+**Key finding**: v0.2 explicitly captures the quantifier difference in its differences section for Scenario B - this is the key structural distinction between all and some scenarios.
+
+## Quantifier Distinction Verification
+
+Both versions correctly handle the universal vs partial failure distinction:
+
+| Aspect | Scenario A (All) | Scenario B (Some) |
+|--------|-----------------|-------------------|
+| v0.1 observations | 1 (universal) | 2 (login OK + download fail) |
+| v0.1 contradictions | 0 (expected) | 0 (expected) |
+| v0.2 primaryType | observed_problem | observed_problem |
+| v0.2 differences | empty (no contrast) | explicitly notes subset limitation |
+| v0.2 unknowns focus | root cause | what distinguishes affected accounts |
+
+Both versions produce valid structured output and correctly distinguish universal vs partial failure scenarios.
+
+## Prompt Fix Summary
+
+The v0.2 prompt template (prompts/reconstruct-v0.2.md) was updated to include an explicit JSON output schema section that:
+1. Specifies exact camelCase key names matching the Zod schema
+2. Lists all valid enum values for primaryType and reasoningModes
+3. Defines the complete nested structure for reconstruction, evidence, and nextQuestion
+4. Includes critical rules preventing snake_case keys or invented top-level fields
+
+Before fix: Model output had input_classification, reasoning_mode, anchors - all invalid per Zod schema -> validationStatus: invalid
+After fix: Model output has inputClassification, reconstruction, evidence, nextQuestion with correct nested structure -> validationStatus: valid
diff --git a/prompts/reconstruct-v0.2.md b/prompts/reconstruct-v0.2.md
new file mode 100644
index 0000000..6d0f7dd
--- /dev/null
+++ b/prompts/reconstruct-v0.2.md
@@ -0,0 +1,122 @@
+You are a neutral analyst performing evidence-based situation reconstruction.
+
+## Rules
+
+1. Do NOT invent facts, context or causes. Only include information present in the scenario or clearly implied.
+2. First determine what kind of input has been supplied. Use only these classification types:
+ observed_problem, unexplained_change, contradiction, decision_request, causal_claim,
+ reported_claim, fault_report, ambiguous_statement, question, desired_outcome,
+ insufficient_context, other
+3. Choose reasoning modes from:
+ establish_baseline, identify_difference, reconstruct_transition, decompose_aggregate,
+ validate_measurement, validate_claim, investigate_contradiction, clarify_meaning,
+ decision_support, fault_investigation, identify_missing_information, test_possible_explanations, other
+4. Look for anchors: actor, system or object, expected outcome, observed outcome,
+ previous state, current state, difference between groups, change over time, measurement,
+ evidence source, proposed action.
+5. Identify meaningful differences (e.g., some succeed while others fail; revenue rises while cash falls).
+6. Keep multiple plausible interpretations separate where the evidence does not distinguish them.
+7. Distinguish: what was said / what it may mean / why it may have been said.
+8. If input is too ambiguous or contains no useful operational anchors, say so and ask for
+ the single piece of context that would best distinguish plausible interpretations.
+
+## Confidence scale
+
+- low — weak evidence, speculation, or missing information
+- medium — reasonable inference from available evidence
+- high — strong evidence, direct observation, or confirmed fact
+
+## Importance scale (evidence records)
+
+- incidental — minor detail, unlikely to affect conclusions
+- supporting — adds context but not critical
+- important — materially affects understanding of the situation
+- critical — essential to resolving the situation; without it conclusions cannot be drawn
+
+## Expected information value (next question)
+
+- low — marginally useful even if answered
+- medium — meaningfully clarifies the situation
+- high — would significantly distinguish between plausible explanations or fill a gap in understanding
+
+## Next question selection criteria
+
+Prefer questions that:
+- clarify a major difference
+- establish a baseline
+- explain an important transition
+- test an unsupported claim
+- distinguish between plausible explanations
+- request measurable evidence
+- identify who or what is affected
+- establish timing
+
+Avoid questions that:
+- have already been answered
+- assume a cause
+- jump to a solution
+- ask about motive before the observable situation is understood
+- focus on incidental wording
+- are too broad to produce useful information
+- combine many unrelated questions
+
+## Output format — return this exact JSON structure
+
+Return a JSON object with exactly these four top-level keys (use **camelCase**):
+
+```json
+{
+ "inputClassification": {
+ "primaryType": "",
+ "secondaryTypes": [""],
+ "reasoningModes": [""],
+ "classificationReason": "",
+ "confidence": ""
+ },
+ "reconstruction": {
+ "summary": "",
+ "actors": [{"id": "", "description": "...", "confidence": ""}],
+ "systemsOrObjects": [{"id": "", "description": "...", "confidence": ""}],
+ "expectedStates": [{"id": "...", "description": "...", "confidence": ""}],
+ "observedStates": [{"id": "...", "description": "...", "confidence": ""}],
+ "differences": [{"id": "...", "description": "...", "confidence": ""}],
+ "knownTransitions": [{"id": "...", "description": "...", "confidence": "", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}],
+ "unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "", "entity": "...", "previousState": "...", "currentState": "..."}],
+ "contradictions": [{"id": "...", "description": "...", "confidence": ""}],
+ "importantUnknowns": [{"id": "...", "description": "...", "confidence": ""}],
+ "plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": [""], "assumptionsRequired": [], "confidence": ""}]
+ },
+ "evidence": [
+ {
+ "id": "",
+ "description": "...",
+ "evidenceType": "",
+ "source": "",
+ "attribution": null,
+ "confidence": "",
+ "importance": ""
+ }
+ ],
+ "nextQuestion": {
+ "id": "",
+ "question": "",
+ "targets": [""],
+ "reason": "",
+ "expectedInformationValue": "",
+ "reasoningMode": ""
+ }
+}
+```
+
+CRITICAL RULES for JSON output:
+1. Use **exactly** the key names shown above (camelCase, no snake_case).
+2. The four top-level keys must be: `inputClassification`, `reconstruction`, `evidence`, `nextQuestion`.
+3. Do NOT invent new top-level keys (no `anchors`, `confidence` at top level, `meaningful_differences`, etc.).
+4. Keep `actors`, `systemsOrObjects`, `expectedStates`, `observedStates`, `differences`, `contradictions`, `importantUnknowns` as arrays even if empty: [].
+5. Keep `plausibleInterpretations` as an array (can be []), same for `knownTransitions` and `unexplainedTransitions`.
+6. Each object in arrays must have at least `id`, `description`, `confidence`.
+
+Scenario:
+{{SCENARIO}}
+
+Return ONLY the JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.
diff --git a/provider-debug-results/2026-08-01T06-35-55.897Z/debug-results.json b/provider-debug-results/2026-08-01T06-35-55.897Z/debug-results.json
new file mode 100644
index 0000000..a19f9f7
--- /dev/null
+++ b/provider-debug-results/2026-08-01T06-35-55.897Z/debug-results.json
@@ -0,0 +1,8 @@
+{
+ "qwen-claude:latest": {
+ "availability": "NOT_AVAILABLE_ON_SERVER"
+ },
+ "qwen3.6:35b-a3b": {
+ "availability": "NOT_AVAILABLE_ON_SERVER"
+ }
+}
\ No newline at end of file
diff --git a/provider-debug-results/2026-08-01T06-37-17.829Z/debug-results.json b/provider-debug-results/2026-08-01T06-37-17.829Z/debug-results.json
new file mode 100644
index 0000000..5b11bb7
--- /dev/null
+++ b/provider-debug-results/2026-08-01T06-37-17.829Z/debug-results.json
@@ -0,0 +1,142 @@
+{
+ "qwen-claude:latest": {
+ "test_A_qwen_claude": {
+ "testDescription": "Plain instruction test — should return CHAT_WORKS",
+ "endpoint": "/api/chat",
+ "format": "json",
+ "hasSystemMessage": true,
+ "httpStatus": 200,
+ "messageContentType": "string",
+ "messageContentLength": 30,
+ "thinkingPresent": true,
+ "parsedContentKeys": [
+ "response"
+ ],
+ "looksLikeStatusAck": false,
+ "rawPreview": "{\n \"response\": \"CHAT_WORKS\"\n}"
+ },
+ "test_A_qwen_claude_generate": {
+ "endpoint": "/api/generate",
+ "httpStatus": 200,
+ "responseFirst200": "CHAT_WORKS",
+ "responseLooksLikeStructuredJSON": false,
+ "rawPreview": "CHAT_WORKS"
+ },
+ "test_B_qwen_claude": {
+ "testDescription": "JSON schema test — should return exact object",
+ "endpoint": "/api/chat",
+ "format": "json",
+ "hasSystemMessage": false,
+ "httpStatus": 200,
+ "messageContentType": "string",
+ "messageContentLength": 38,
+ "thinkingPresent": true,
+ "parsedContentKeys": [
+ "message"
+ ],
+ "looksLikeStatusAck": false,
+ "rawPreview": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}"
+ },
+ "test_B_qwen_claude_generate": {
+ "endpoint": "/api/generate",
+ "httpStatus": 200,
+ "responseFirst200": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}",
+ "responseLooksLikeStructuredJSON": true,
+ "rawPreview": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}"
+ },
+ "test_C_qwen_claude": {
+ "testDescription": "Minimal reconstruction schema — structured output test",
+ "endpoint": "/api/chat",
+ "format": "json",
+ "hasSystemMessage": false,
+ "httpStatus": 200,
+ "messageContentType": "string",
+ "messageContentLength": 1145,
+ "thinkingPresent": true,
+ "parsedContentKeys": [
+ "analysis",
+ "meaningful_difference",
+ "next_question"
+ ],
+ "looksLikeStatusAck": false,
+ "rawPreview": "{\"analysis\":\"The workflow breaks into two distinct stages: authentication (login) and resource access (invoice download). Since login succeeds for the affected users, identity verification and session creation are functioning correctly. The failure occurs downstream in the layer responsible for loca"
+ },
+ "test_C_qwen_claude_generate": {
+ "endpoint": "/api/generate",
+ "httpStatus": 200,
+ "responseFirst200": "**Analysis of the Situation**\nThe issue represents a partial, post-authentication failure: identity verification works for a subset of users, but access to or delivery of a specific resource (invoices",
+ "responseLooksLikeStructuredJSON": false,
+ "rawPreview": "**Analysis of the Situation**\nThe issue represents a partial, post-authentication failure: identity verification works for a subset of users, but access to or delivery of a specific resource (invoices"
+ }
+ },
+ "qwen3.6:35b-a3b": {
+ "test_A_qwen___": {
+ "testDescription": "Plain instruction test — should return CHAT_WORKS",
+ "endpoint": "/api/chat",
+ "format": "json",
+ "hasSystemMessage": true,
+ "httpStatus": 200,
+ "messageContentType": "string",
+ "messageContentLength": 30,
+ "thinkingPresent": true,
+ "parsedContentKeys": [
+ "response"
+ ],
+ "looksLikeStatusAck": false,
+ "rawPreview": "{\n \"response\": \"CHAT_WORKS\"\n}"
+ },
+ "test_A_qwen____generate": {
+ "endpoint": "/api/generate",
+ "httpStatus": 200,
+ "responseFirst200": "CHAT_WORKS",
+ "responseLooksLikeStructuredJSON": false,
+ "rawPreview": "CHAT_WORKS"
+ },
+ "test_B_qwen___": {
+ "testDescription": "JSON schema test — should return exact object",
+ "endpoint": "/api/chat",
+ "format": "json",
+ "hasSystemMessage": false,
+ "httpStatus": 200,
+ "messageContentType": "string",
+ "messageContentLength": 38,
+ "thinkingPresent": true,
+ "parsedContentKeys": [
+ "message"
+ ],
+ "looksLikeStatusAck": false,
+ "rawPreview": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}"
+ },
+ "test_B_qwen____generate": {
+ "endpoint": "/api/generate",
+ "httpStatus": 200,
+ "responseFirst200": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}",
+ "responseLooksLikeStructuredJSON": true,
+ "rawPreview": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}"
+ },
+ "test_C_qwen___": {
+ "testDescription": "Minimal reconstruction schema — structured output test",
+ "endpoint": "/api/chat",
+ "format": "json",
+ "hasSystemMessage": false,
+ "httpStatus": 200,
+ "messageContentType": "string",
+ "messageContentLength": 843,
+ "thinkingPresent": true,
+ "parsedContentKeys": [
+ "analysis",
+ "meaningful_difference",
+ "next_question"
+ ],
+ "looksLikeStatusAck": false,
+ "rawPreview": "{\"analysis\": \"The scenario indicates that authentication is functioning correctly, but a downstream access or resource-retrieval step is failing for a subset of users. This separates the problem from credential or session validity and points toward post-login factors such as authorization scope, doc"
+ },
+ "test_C_qwen____generate": {
+ "endpoint": "/api/generate",
+ "httpStatus": 200,
+ "responseFirst200": "**Meaningful Difference:** \nSuccessful login verifies *account-level authentication and baseline platform access*, while downloading an invoice requires *invoice-specific data availability, retrieval",
+ "responseLooksLikeStructuredJSON": false,
+ "rawPreview": "**Meaningful Difference:** \nSuccessful login verifies *account-level authentication and baseline platform access*, while downloading an invoice requires *invoice-specific data availability, retrieval"
+ }
+ }
+}
\ No newline at end of file
diff --git a/scripts/debug-ollama-direct.mjs b/scripts/debug-ollama-direct.mjs
new file mode 100644
index 0000000..6fb86c8
--- /dev/null
+++ b/scripts/debug-ollama-direct.mjs
@@ -0,0 +1,218 @@
+/**
+ * Debug script: send raw Ollama requests directly, bypassing the application provider.
+ * Tests /api/chat with format:json and captures request payloads + raw responses.
+ */
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+const BASE_URL = process.env.OLLAMA_BASE_URL || "http://localhost:11434";
+const TIMESTAMP = new Date().toISOString().replace(/[/:]/g, "-");
+const RESULTS_DIR = join(__dirname, "..", "provider-debug-results", TIMESTAMP);
+
+mkdirSync(RESULTS_DIR, { recursive: true });
+
+// ============================================================
+// Test cases
+// ============================================================
+
+const MODEL_A = "qwen-claude:latest";
+const MODEL_B = "qwen3.6:35b-a3b";
+
+function getModelList() {
+ // Check which models are available locally (not via Ollama server)
+ return { A: MODEL_A, B: MODEL_B };
+}
+
+// Test A: Simple text reply to verify model responds normally
+const TEST_A = {
+ label: "A",
+ description: "Plain instruction test — should return CHAT_WORKS",
+ system: "You are a normal assistant. Follow the user instruction exactly.",
+ user: "Reply with exactly: CHAT_WORKS",
+};
+
+// Test B: Explicit JSON schema via format field
+const TEST_B = {
+ label: "B",
+ description: "JSON schema test — should return exact object",
+ system: null, // uses messages only with format
+ user: 'Return exactly: {"message": "STRUCTURED_OUTPUT_WORKS"}',
+};
+
+// Test C: Minimal reconstruction-style schema
+const TEST_C = {
+ label: "C",
+ description: "Minimal reconstruction schema — structured output test",
+ system: null,
+ user: "Analyse this situation without solving it: Some customers can log in but cannot download invoices. Identify the meaningful difference and ask one useful next question.",
+};
+
+const ALL_TESTS = [TEST_A, TEST_B, TEST_C];
+
+// ============================================================
+// Helper functions
+// ============================================================
+
+async function runChatWithFormat(model, messages, format) {
+ const body = { model, messages, stream: false, format };
+ const res = await fetch(`${BASE_URL}/api/chat`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify(body),
+ });
+
+ const rawText = await res.text();
+ let parsed = null;
+ try { parsed = JSON.parse(rawText); } catch {}
+
+ return {
+ status: res.status,
+ statusText: res.statusText,
+ requestPayload: body,
+ rawResponseText: rawText.slice(0, 5000),
+ parsedResponse: parsed,
+ messageContent: parsed?.message?.content ?? null,
+ thinkingLength: (parsed?.message?.thinking || "").length,
+ messageContentType: typeof parsed?.message?.content,
+ responseField: parsed?.response,
+ };
+}
+
+async function runGenerate(model, prompt) {
+ const body = { model, prompt, stream: false };
+ const res = await fetch(`${BASE_URL}/api/generate`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify(body),
+ });
+
+ const rawText = await res.text();
+ let parsed = null;
+ try { parsed = JSON.parse(rawText); } catch {}
+
+ return {
+ status: res.status,
+ requestPayload: body,
+ rawResponseText: rawText.slice(0, 5000),
+ parsedResponse: parsed,
+ responseField: typeof parsed?.response === "string" ? parsed.response : JSON.stringify(parsed),
+ responseFirst200: (parsed?.response || "").slice(0, 200),
+ };
+}
+
+// ============================================================
+// Run tests
+// ============================================================
+
+const results = {};
+
+for (const model of [MODEL_A, MODEL_B]) {
+ console.log(`\n=== Testing model: ${model} ===`);
+ results[model] = {};
+
+ // Check if model is available locally
+ let available = false;
+ try {
+ const tagsRes = await fetch(`${BASE_URL}/api/tags`);
+ const tagsData = await tagsRes.json();
+ available = tagsData.models?.some(m => m.name.includes(model.split(":")[0]));
+ } catch (e) {
+ console.log(` Warning: could not check model availability: ${e.message}`);
+ }
+
+ if (!available) {
+ results[model].availability = "NOT_AVAILABLE_ON_SERVER";
+ console.log(` -> Model ${model} not found on server, skipping`);
+ continue;
+ }
+
+ console.log(` -> Model available on server\n`);
+
+ for (const test of ALL_TESTS) {
+ const testKey = `test_${test.label}_${model.split(":")[0].replace(/[^a-zA-Z]/g, "_")}`;
+ console.log(` Running Test ${test.label}: ${test.description}`);
+
+ // Chat with format:json
+ let chatResult;
+ try {
+ const messages = [];
+ if (test.system) {
+ messages.push({ role: "system", content: test.system });
+ }
+ messages.push({ role: "user", content: test.user });
+
+ chatResult = await runChatWithFormat(model, messages, "json");
+
+ // Try to extract JSON from message.content
+ let extractedJson = null;
+ if (typeof chatResult.messageContent === "string") {
+ try {
+ extractedJson = JSON.parse(chatResult.messageContent);
+ } catch {}
+ }
+
+ results[model][testKey] = {
+ testDescription: test.description,
+ endpoint: "/api/chat",
+ format: "json",
+ hasSystemMessage: !!test.system,
+ httpStatus: chatResult.status,
+ messageContentType: chatResult.messageContentType,
+ messageContentLength: chatResult.messageContent?.length || 0,
+ thinkingPresent: chatResult.thinkingLength > 0,
+ parsedContentKeys: extractedJson ? Object.keys(extractedJson) : null,
+ // If content looks like a status acknowledgment
+ looksLikeStatusAck: typeof chatResult.messageContent === "string" &&
+ (chatResult.messageContent.includes('"status"') || chatResult.messageContent.includes('"state"')),
+ rawPreview: chatResult.messageContent?.slice(0, 300) ?? "(none)",
+ };
+
+ const status = extractedJson ? "JSON_OK" : (chatResult.messageContent ? "TEXT_RESPONSE" : "EMPTY");
+ console.log(` -> ${status} (HTTP ${chatResult.status}, content type: ${chatResult.messageContentType})`);
+ if (extractedJson) {
+ console.log(` JSON keys: ${Object.keys(extractedJson).join(", ")}`);
+ } else if (chatResult.messageContent) {
+ console.log(` Content preview: ${(typeof chatResult.messageContent === "string" ? chatResult.messageContent : String(chatResult.messageContent)).slice(0, 150)}...`);
+ }
+
+ } catch (e) {
+ results[model][testKey] = { error: e.message };
+ console.log(` -> ERROR: ${e.message}`);
+ }
+
+ // Generate (fallback test)
+ let generateResult;
+ try {
+ const generatePrompt = test.system ? `${test.system}\n\n${test.user}` : test.user;
+ generateResult = await runGenerate(model, generatePrompt);
+
+ results[model][`${testKey}_generate`] = {
+ endpoint: "/api/generate",
+ httpStatus: generateResult.status,
+ responseFirst200: generateResult.responseFirst200,
+ responseLooksLikeStructuredJSON: generateResult.responseField?.trim().startsWith("{"),
+ rawPreview: generateResult.responseFirst200,
+ };
+
+ const isJson = generateResult.responseField?.trim().startsWith("{") ? "JSON_START" : "NOT_JSON";
+ console.log(` -> ${isJson} (HTTP ${generateResult.status})`);
+
+ } catch (e) {
+ results[model][`${testKey}_generate`] = { error: e.message };
+ console.log(` -> GENERATE ERROR: ${e.message}`);
+ }
+
+ console.log();
+ }
+}
+
+// ============================================================
+// Save results
+// ============================================================
+
+const saveFile = join(RESULTS_DIR, "debug-results.json");
+writeFileSync(saveFile, JSON.stringify(results, null, 2));
+console.log(`\nResults saved to: ${saveFile}`);
diff --git a/tests-results/evaluation-2026-07-31T18-14-22.json b/tests-results/evaluation-2026-07-31T18-14-22.json
new file mode 100644
index 0000000..416f672
--- /dev/null
+++ b/tests-results/evaluation-2026-07-31T18-14-22.json
@@ -0,0 +1,653 @@
+{
+ "timestamp": "2026-07-31T18:14:22.632Z",
+ "provider": "mock",
+ "promptVersion": "v0.2",
+ "casesRun": 35,
+ "summary": {
+ "schemaValidityRate": "100.0%",
+ "classificationMatchRate": "28.6%",
+ "reasoningModeMatchRate": "48.6%",
+ "requiredConceptMatchRate": "0.0%",
+ "unsupportedInferenceFailures": "0",
+ "averageResponseDurationMs": "1",
+ "fullPassRate": "28.6%"
+ },
+ "testCaseResults": [
+ {
+ "id": "tc-001",
+ "input": "All customers cannot download their invoices.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 4,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-002",
+ "input": "Some customers cannot download their invoices.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-003",
+ "input": "Complaints increased by 35%.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-004",
+ "input": "Complaints increased by 35% while production increased by 40%.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-005",
+ "input": "Sales are falling.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-006",
+ "input": "Sales fell sharply immediately after the price increase.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-007",
+ "input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-008",
+ "input": "Revenue from the premium tier dropped while total revenue grew.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-009",
+ "input": "We need to improve our customer retention rate.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-010",
+ "input": "The system latency went from 200ms to 5 seconds on Tuesday.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-011",
+ "input": "The new release should fix the login issue.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-012",
+ "input": "I think therefore I am.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-013",
+ "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-014",
+ "input": "The warehouse manager reported that inventory counts don't match the system.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-015",
+ "input": "We've seen a 35% increase in customer complaints.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-016",
+ "input": "The number of active users increased by 500%, from 4 to 2,001.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-017",
+ "input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-018",
+ "input": "The manufacturing team needs better quality control.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-019",
+ "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-020",
+ "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-021",
+ "input": "Production output was 1,200 units last month and 1,180 units this month.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-022",
+ "input": "The CFO reported that the company's cash position is healthy.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-023",
+ "input": "We have enough funding to operate for 18 months.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-024",
+ "input": "The new feature was deployed at 3am and user complaints tripled the next day.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-025",
+ "input": "We need to launch a mobile app to capture market share.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-026",
+ "input": "The system has been running for 90 days without failure since the migration.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-027",
+ "input": "No one has submitted the required compliance report despite multiple reminders.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-028",
+ "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-029",
+ "input": "We should implement the new CRM because our competitors have one.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-030",
+ "input": "The server response time was acceptable last quarter but degraded this month.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-031",
+ "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-032",
+ "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-033",
+ "input": "The database schema was changed on Friday but the reports are still working.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-034",
+ "input": "Some team members say the new process is better while others say it's slower.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-035",
+ "input": "The application works fine on Chrome but not on Safari.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "identify_difference",
+ "investigate_contradiction"
+ ],
+ "errors": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests-results/evaluation-2026-07-31T18-16-32.json b/tests-results/evaluation-2026-07-31T18-16-32.json
new file mode 100644
index 0000000..4fa8dfd
--- /dev/null
+++ b/tests-results/evaluation-2026-07-31T18-16-32.json
@@ -0,0 +1,652 @@
+{
+ "timestamp": "2026-07-31T18:16:32.255Z",
+ "provider": "mock",
+ "promptVersion": "v0.2",
+ "casesRun": 35,
+ "summary": {
+ "schemaValidityRate": "100.0%",
+ "classificationMatchRate": "37.1%",
+ "reasoningModeMatchRate": "71.4%",
+ "requiredConceptMatchRate": "0.0%",
+ "unsupportedInferenceFailures": "0",
+ "averageResponseDurationMs": "1",
+ "fullPassRate": "37.1%"
+ },
+ "testCaseResults": [
+ {
+ "id": "tc-001",
+ "input": "All customers cannot download their invoices.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 4,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-002",
+ "input": "Some customers cannot download their invoices.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-003",
+ "input": "Complaints increased by 35%.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-004",
+ "input": "Complaints increased by 35% while production increased by 40%.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-005",
+ "input": "Sales are falling.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-006",
+ "input": "Sales fell sharply immediately after the price increase.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-007",
+ "input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-008",
+ "input": "Revenue from the premium tier dropped while total revenue grew.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-009",
+ "input": "We need to improve our customer retention rate.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-010",
+ "input": "The system latency went from 200ms to 5 seconds on Tuesday.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-011",
+ "input": "The new release should fix the login issue.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-012",
+ "input": "I think therefore I am.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "ambiguous_statement",
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-013",
+ "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "ambiguous_statement",
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-014",
+ "input": "The warehouse manager reported that inventory counts don't match the system.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-015",
+ "input": "We've seen a 35% increase in customer complaints.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-016",
+ "input": "The number of active users increased by 500%, from 4 to 2,001.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-017",
+ "input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-018",
+ "input": "The manufacturing team needs better quality control.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-019",
+ "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-020",
+ "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-021",
+ "input": "Production output was 1,200 units last month and 1,180 units this month.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-022",
+ "input": "The CFO reported that the company's cash position is healthy.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-023",
+ "input": "We have enough funding to operate for 18 months.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-024",
+ "input": "The new feature was deployed at 3am and user complaints tripled the next day.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-025",
+ "input": "We need to launch a mobile app to capture market share.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-026",
+ "input": "The system has been running for 90 days without failure since the migration.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-027",
+ "input": "No one has submitted the required compliance report despite multiple reminders.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-028",
+ "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-029",
+ "input": "We should implement the new CRM because our competitors have one.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-030",
+ "input": "The server response time was acceptable last quarter but degraded this month.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-031",
+ "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-032",
+ "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-033",
+ "input": "The database schema was changed on Friday but the reports are still working.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-034",
+ "input": "Some team members say the new process is better while others say it's slower.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "investigate_contradiction",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-035",
+ "input": "The application works fine on Chrome but not on Safari.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests-results/evaluation-2026-07-31T18-23-11.json b/tests-results/evaluation-2026-07-31T18-23-11.json
new file mode 100644
index 0000000..14d293e
--- /dev/null
+++ b/tests-results/evaluation-2026-07-31T18-23-11.json
@@ -0,0 +1,633 @@
+{
+ "timestamp": "2026-07-31T18:23:11.669Z",
+ "provider": "mock",
+ "promptVersion": "v0.2",
+ "casesRun": 35,
+ "summary": {
+ "schemaValidityRate": "100.0%",
+ "classificationMatchRate": "48.6%",
+ "reasoningModeMatchRate": "48.6%",
+ "requiredConceptMatchRate": "0.0%",
+ "unsupportedInferenceFailures": "0",
+ "averageResponseDurationMs": "1",
+ "fullPassRate": "48.6%"
+ },
+ "testCaseResults": [
+ {
+ "id": "tc-001",
+ "input": "All customers cannot download their invoices.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 6,
+ "actualPrimaryType": "fault_report",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-002",
+ "input": "Some customers cannot download their invoices.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-003",
+ "input": "Complaints increased by 35%.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "fault_report",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-004",
+ "input": "Complaints increased by 35% while production increased by 40%.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "fault_report",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-005",
+ "input": "Sales are falling.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "fault_report",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-006",
+ "input": "Sales fell sharply immediately after the price increase.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-007",
+ "input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "other",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-008",
+ "input": "Revenue from the premium tier dropped while total revenue grew.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-009",
+ "input": "We need to improve our customer retention rate.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-010",
+ "input": "The system latency went from 200ms to 5 seconds on Tuesday.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-011",
+ "input": "The new release should fix the login issue.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "other",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-012",
+ "input": "I think therefore I am.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "ambiguous_statement",
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-013",
+ "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "ambiguous_statement",
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-014",
+ "input": "The warehouse manager reported that inventory counts don't match the system.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-015",
+ "input": "We've seen a 35% increase in customer complaints.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "fault_report",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-016",
+ "input": "The number of active users increased by 500%, from 4 to 2,001.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "other",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-017",
+ "input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "other",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-018",
+ "input": "The manufacturing team needs better quality control.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "other",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-019",
+ "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "fault_report",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-020",
+ "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-021",
+ "input": "Production output was 1,200 units last month and 1,180 units this month.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "other",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-022",
+ "input": "The CFO reported that the company's cash position is healthy.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-023",
+ "input": "We have enough funding to operate for 18 months.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "other",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-024",
+ "input": "The new feature was deployed at 3am and user complaints tripled the next day.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "fault_report",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-025",
+ "input": "We need to launch a mobile app to capture market share.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-026",
+ "input": "The system has been running for 90 days without failure since the migration.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "other",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-027",
+ "input": "No one has submitted the required compliance report despite multiple reminders.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "fault_report",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-028",
+ "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "other",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-029",
+ "input": "We should implement the new CRM because our competitors have one.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-030",
+ "input": "The server response time was acceptable last quarter but degraded this month.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-031",
+ "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "fault_report",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-032",
+ "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "other",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-033",
+ "input": "The database schema was changed on Friday but the reports are still working.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "other",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-034",
+ "input": "Some team members say the new process is better while others say it's slower.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "investigate_contradiction",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-035",
+ "input": "The application works fine on Chrome but not on Safari.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "other",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests-results/evaluation-2026-07-31T18-34-17.json b/tests-results/evaluation-2026-07-31T18-34-17.json
new file mode 100644
index 0000000..025baf1
--- /dev/null
+++ b/tests-results/evaluation-2026-07-31T18-34-17.json
@@ -0,0 +1,639 @@
+{
+ "timestamp": "2026-07-31T18:34:17.855Z",
+ "provider": "mock",
+ "promptVersion": "v0.2",
+ "casesRun": 35,
+ "summary": {
+ "schemaValidityRate": "100.0%",
+ "classificationMatchRate": "82.9%",
+ "reasoningModeMatchRate": "51.4%",
+ "requiredConceptMatchRate": "0.0%",
+ "unsupportedInferenceFailures": "0",
+ "averageResponseDurationMs": "1",
+ "fullPassRate": "82.9%"
+ },
+ "testCaseResults": [
+ {
+ "id": "tc-001",
+ "input": "All customers cannot download their invoices.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 5,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-002",
+ "input": "Some customers cannot download their invoices.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 2,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-003",
+ "input": "Complaints increased by 35%.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-004",
+ "input": "Complaints increased by 35% while production increased by 40%.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-005",
+ "input": "Sales are falling.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-006",
+ "input": "Sales fell sharply immediately after the price increase.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-007",
+ "input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "investigate_contradiction",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-008",
+ "input": "Revenue from the premium tier dropped while total revenue grew.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-009",
+ "input": "We need to improve our customer retention rate.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-010",
+ "input": "The system latency went from 200ms to 5 seconds on Tuesday.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-011",
+ "input": "The new release should fix the login issue.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-012",
+ "input": "I think therefore I am.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "ambiguous_statement",
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-013",
+ "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "ambiguous_statement",
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-014",
+ "input": "The warehouse manager reported that inventory counts don't match the system.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-015",
+ "input": "We've seen a 35% increase in customer complaints.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-016",
+ "input": "The number of active users increased by 500%, from 4 to 2,001.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-017",
+ "input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-018",
+ "input": "The manufacturing team needs better quality control.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-019",
+ "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-020",
+ "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-021",
+ "input": "Production output was 1,200 units last month and 1,180 units this month.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-022",
+ "input": "The CFO reported that the company's cash position is healthy.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-023",
+ "input": "We have enough funding to operate for 18 months.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-024",
+ "input": "The new feature was deployed at 3am and user complaints tripled the next day.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-025",
+ "input": "We need to launch a mobile app to capture market share.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-026",
+ "input": "The system has been running for 90 days without failure since the migration.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-027",
+ "input": "No one has submitted the required compliance report despite multiple reminders.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-028",
+ "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-029",
+ "input": "We should implement the new CRM because our competitors have one.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-030",
+ "input": "The server response time was acceptable last quarter but degraded this month.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-031",
+ "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-032",
+ "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-033",
+ "input": "The database schema was changed on Friday but the reports are still working.",
+ "pass": false,
+ "schemaValid": true,
+ "classificationMatch": false,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-034",
+ "input": "Some team members say the new process is better while others say it's slower.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-035",
+ "input": "The application works fine on Chrome but not on Safari.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests-results/evaluation-2026-07-31T18-59-37.json b/tests-results/evaluation-2026-07-31T18-59-37.json
new file mode 100644
index 0000000..74d39ca
--- /dev/null
+++ b/tests-results/evaluation-2026-07-31T18-59-37.json
@@ -0,0 +1,644 @@
+{
+ "timestamp": "2026-07-31T18:59:37.227Z",
+ "provider": "mock",
+ "promptVersion": "v0.2",
+ "casesRun": 35,
+ "summary": {
+ "schemaValidityRate": "100.0%",
+ "classificationMatchRate": "100.0%",
+ "reasoningModeMatchRate": "57.1%",
+ "requiredConceptMatchRate": "0.0%",
+ "unsupportedInferenceFailures": "0",
+ "averageResponseDurationMs": "0",
+ "fullPassRate": "100.0%"
+ },
+ "testCaseResults": [
+ {
+ "id": "tc-001",
+ "input": "All customers cannot download their invoices.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 4,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-002",
+ "input": "Some customers cannot download their invoices.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-003",
+ "input": "Complaints increased by 35%.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-004",
+ "input": "Complaints increased by 35% while production increased by 40%.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-005",
+ "input": "Sales are falling.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-006",
+ "input": "Sales fell sharply immediately after the price increase.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-007",
+ "input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "investigate_contradiction",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-008",
+ "input": "Revenue from the premium tier dropped while total revenue grew.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-009",
+ "input": "We need to improve our customer retention rate.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-010",
+ "input": "The system latency went from 200ms to 5 seconds on Tuesday.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-011",
+ "input": "The new release should fix the login issue.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-012",
+ "input": "I think therefore I am.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "ambiguous_statement",
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-013",
+ "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "ambiguous_statement",
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-014",
+ "input": "The warehouse manager reported that inventory counts don't match the system.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-015",
+ "input": "We've seen a 35% increase in customer complaints.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-016",
+ "input": "The number of active users increased by 500%, from 4 to 2,001.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-017",
+ "input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "investigate_contradiction",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-018",
+ "input": "The manufacturing team needs better quality control.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-019",
+ "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-020",
+ "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-021",
+ "input": "Production output was 1,200 units last month and 1,180 units this month.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-022",
+ "input": "The CFO reported that the company's cash position is healthy.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-023",
+ "input": "We have enough funding to operate for 18 months.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-024",
+ "input": "The new feature was deployed at 3am and user complaints tripled the next day.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-025",
+ "input": "We need to launch a mobile app to capture market share.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-026",
+ "input": "The system has been running for 90 days without failure since the migration.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-027",
+ "input": "No one has submitted the required compliance report despite multiple reminders.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-028",
+ "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-029",
+ "input": "We should implement the new CRM because our competitors have one.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-030",
+ "input": "The server response time was acceptable last quarter but degraded this month.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-031",
+ "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-032",
+ "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-033",
+ "input": "The database schema was changed on Friday but the reports are still working.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-034",
+ "input": "Some team members say the new process is better while others say it's slower.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-035",
+ "input": "The application works fine on Chrome but not on Safari.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests-results/evaluation-2026-07-31T19-00-09.json b/tests-results/evaluation-2026-07-31T19-00-09.json
new file mode 100644
index 0000000..7d9ce91
--- /dev/null
+++ b/tests-results/evaluation-2026-07-31T19-00-09.json
@@ -0,0 +1,644 @@
+{
+ "timestamp": "2026-07-31T19:00:09.575Z",
+ "provider": "mock",
+ "promptVersion": "v0.2",
+ "casesRun": 35,
+ "summary": {
+ "schemaValidityRate": "100.0%",
+ "classificationMatchRate": "100.0%",
+ "reasoningModeMatchRate": "57.1%",
+ "requiredConceptMatchRate": "0.0%",
+ "unsupportedInferenceFailures": "0",
+ "averageResponseDurationMs": "1",
+ "fullPassRate": "100.0%"
+ },
+ "testCaseResults": [
+ {
+ "id": "tc-001",
+ "input": "All customers cannot download their invoices.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 6,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-002",
+ "input": "Some customers cannot download their invoices.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 2,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-003",
+ "input": "Complaints increased by 35%.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-004",
+ "input": "Complaints increased by 35% while production increased by 40%.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-005",
+ "input": "Sales are falling.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-006",
+ "input": "Sales fell sharply immediately after the price increase.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-007",
+ "input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "investigate_contradiction",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-008",
+ "input": "Revenue from the premium tier dropped while total revenue grew.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-009",
+ "input": "We need to improve our customer retention rate.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-010",
+ "input": "The system latency went from 200ms to 5 seconds on Tuesday.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-011",
+ "input": "The new release should fix the login issue.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-012",
+ "input": "I think therefore I am.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "ambiguous_statement",
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-013",
+ "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "ambiguous_statement",
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-014",
+ "input": "The warehouse manager reported that inventory counts don't match the system.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-015",
+ "input": "We've seen a 35% increase in customer complaints.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-016",
+ "input": "The number of active users increased by 500%, from 4 to 2,001.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-017",
+ "input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "investigate_contradiction",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-018",
+ "input": "The manufacturing team needs better quality control.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-019",
+ "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-020",
+ "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-021",
+ "input": "Production output was 1,200 units last month and 1,180 units this month.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-022",
+ "input": "The CFO reported that the company's cash position is healthy.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-023",
+ "input": "We have enough funding to operate for 18 months.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-024",
+ "input": "The new feature was deployed at 3am and user complaints tripled the next day.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-025",
+ "input": "We need to launch a mobile app to capture market share.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-026",
+ "input": "The system has been running for 90 days without failure since the migration.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-027",
+ "input": "No one has submitted the required compliance report despite multiple reminders.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-028",
+ "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-029",
+ "input": "We should implement the new CRM because our competitors have one.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-030",
+ "input": "The server response time was acceptable last quarter but degraded this month.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-031",
+ "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-032",
+ "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-033",
+ "input": "The database schema was changed on Friday but the reports are still working.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-034",
+ "input": "Some team members say the new process is better while others say it's slower.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ },
+ {
+ "id": "tc-035",
+ "input": "The application works fine on Chrome but not on Safari.",
+ "pass": true,
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "requiredConceptMatch": false,
+ "unsupportedInferenceAbsence": true,
+ "nextQuestionPresent": true,
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "errors": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests-results/evaluation-2026-08-01T05-17-18.json b/tests-results/evaluation-2026-08-01T05-17-18.json
new file mode 100644
index 0000000..bda85c4
--- /dev/null
+++ b/tests-results/evaluation-2026-08-01T05-17-18.json
@@ -0,0 +1,1735 @@
+{
+ "timestamp": "2026-08-01T05:17:18.969Z",
+ "provider": "mock",
+ "promptVersion": "v0.2",
+ "casesRun": 35,
+ "summary": {
+ "technical": {
+ "schemaValidityRate": "100.0%",
+ "classificationMatchRate": "100.0%",
+ "nextQuestionPresentRate": "100.0%",
+ "passRate": "100.0%"
+ },
+ "reasoningQuality": {
+ "requiredConceptMatchRate": "0.0%",
+ "unsupportedInferenceFailures": "0",
+ "passRate": "0.0%"
+ },
+ "combinedPassRate": "0.0%",
+ "averageResponseDurationMs": "1"
+ },
+ "testCaseResults": [
+ {
+ "id": "tc-001",
+ "input": "All customers cannot download their invoices.",
+ "responseDurationMs": 6,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "customers",
+ "found": false
+ },
+ {
+ "concept": "invoices",
+ "found": false
+ },
+ {
+ "concept": "download",
+ "found": false
+ },
+ {
+ "concept": "access_issue",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-002",
+ "input": "Some customers cannot download their invoices.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "some_customers",
+ "found": false
+ },
+ {
+ "concept": "invoices",
+ "found": false
+ },
+ {
+ "concept": "download",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "root_cause",
+ "absent": true
+ },
+ {
+ "concept": "payment_system_failure",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-003",
+ "input": "Complaints increased by 35%.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "complaints",
+ "found": true
+ },
+ {
+ "concept": "increase",
+ "found": false
+ },
+ {
+ "concept": "35_percent",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cause_of_complaints",
+ "absent": true
+ },
+ {
+ "concept": "customer_dissatisfaction_is_worse",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-004",
+ "input": "Complaints increased by 35% while production increased by 40%.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "complaints_increase",
+ "found": false
+ },
+ {
+ "concept": "production_increase",
+ "found": false
+ },
+ {
+ "concept": "relative_rates",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "production_quality_declined",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-005",
+ "input": "Sales are falling.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "sales_decline",
+ "found": false
+ },
+ {
+ "concept": "direction_negative",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cause_of_fall",
+ "absent": true
+ },
+ {
+ "concept": "competitor_action",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-006",
+ "input": "Sales fell sharply immediately after the price increase.",
+ "responseDurationMs": 2,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "sales_decline",
+ "found": false
+ },
+ {
+ "concept": "price_increase",
+ "found": false
+ },
+ {
+ "concept": "temporal_correlation",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "price_increase_caused_the_fall",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-007",
+ "input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "investigate_contradiction",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "revenue_above_target",
+ "found": false
+ },
+ {
+ "concept": "profit_decline",
+ "found": false
+ },
+ {
+ "concept": "divergence",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cost_overrun_is_the_cause",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-008",
+ "input": "Revenue from the premium tier dropped while total revenue grew.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "premium_tier_decline",
+ "found": false
+ },
+ {
+ "concept": "total_revenue_growth",
+ "found": false
+ },
+ {
+ "concept": "segment_cannibalization_risk",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "pricing_change_occurred",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-009",
+ "input": "We need to improve our customer retention rate.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "retention_improvement_desired",
+ "found": false
+ },
+ {
+ "concept": "current_state_unknown",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "retention_rate_is_low",
+ "absent": true
+ },
+ {
+ "concept": "churn_has_increased",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-010",
+ "input": "The system latency went from 200ms to 5 seconds on Tuesday.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "latency_baseline_200ms",
+ "found": false
+ },
+ {
+ "concept": "latency_spike_5s",
+ "found": false
+ },
+ {
+ "concept": "timestamp_tuesday",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "database_cause",
+ "absent": true
+ },
+ {
+ "concept": "release_cause",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-011",
+ "input": "The new release should fix the login issue.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "proposed_solution",
+ "found": false
+ },
+ {
+ "concept": "login_issue",
+ "found": false
+ },
+ {
+ "concept": "solution_claim",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "login_issue_is_real",
+ "absent": true
+ },
+ {
+ "concept": "release_will_work",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-012",
+ "input": "I think therefore I am.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "philosophical_statement",
+ "found": false
+ },
+ {
+ "concept": "insufficient_operational_context",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "business_problem_exists",
+ "absent": true
+ },
+ {
+ "concept": "actionable_insight_possible",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-013",
+ "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "meta_context",
+ "found": false
+ },
+ {
+ "concept": "testing_hypothesis",
+ "found": false
+ },
+ {
+ "concept": "self_reference",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-014",
+ "input": "The warehouse manager reported that inventory counts don't match the system.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "warehouse_manager_report",
+ "found": false
+ },
+ {
+ "concept": "inventory_mismatch",
+ "found": false
+ },
+ {
+ "concept": "system_discrepancy",
+ "found": false
+ },
+ {
+ "concept": "source_attribution",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "theft_occurred",
+ "absent": true
+ },
+ {
+ "concept": "software_bug",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-015",
+ "input": "We've seen a 35% increase in customer complaints.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "complaints_increase",
+ "found": false
+ },
+ {
+ "concept": "percentage_metric",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "product_quality_declined",
+ "absent": true
+ },
+ {
+ "concept": "customer_satisfaction_drop",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-016",
+ "input": "The number of active users increased by 500%, from 4 to 2,001.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "active_users_metric",
+ "found": false
+ },
+ {
+ "concept": "absolute_vs_relative_growth",
+ "found": false
+ },
+ {
+ "concept": "small_base_problem",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "product_success",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-017",
+ "input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "investigate_contradiction",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "engagement_improvement",
+ "found": false
+ },
+ {
+ "concept": "support_backlog_growth",
+ "found": false
+ },
+ {
+ "concept": "divergent_metrics",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "users_are_angry",
+ "absent": true
+ },
+ {
+ "concept": "product_quality_is_worse",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-018",
+ "input": "The manufacturing team needs better quality control.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "manufacturing_team",
+ "found": false
+ },
+ {
+ "concept": "quality_control_desired",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "quality_is_bad",
+ "absent": true
+ },
+ {
+ "concept": "defect_rate_is_high",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-019",
+ "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "eu_region",
+ "found": false
+ },
+ {
+ "concept": "403_error",
+ "found": false
+ },
+ {
+ "concept": "access_denied",
+ "found": false
+ },
+ {
+ "concept": "geographic_scope",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "gdpr_cause",
+ "absent": true
+ },
+ {
+ "concept": "regulatory_change",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-020",
+ "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "eu_region_subset",
+ "found": false
+ },
+ {
+ "concept": "403_error",
+ "found": false
+ },
+ {
+ "concept": "partial_reachability",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "all_eu_users_affected",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-021",
+ "input": "Production output was 1,200 units last month and 1,180 units this month.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "production_output",
+ "found": false
+ },
+ {
+ "concept": "month_over_month_decline",
+ "found": false
+ },
+ {
+ "concept": "absolute_difference",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "efficiency_loss_occurred",
+ "absent": true
+ },
+ {
+ "concept": "equipment_failure",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-022",
+ "input": "The CFO reported that the company's cash position is healthy.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "cfo_statement",
+ "found": false
+ },
+ {
+ "concept": "cash_position_claim",
+ "found": false
+ },
+ {
+ "concept": "source_attribution_cfo",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cash_is_healthy",
+ "absent": true
+ },
+ {
+ "concept": "financial_stability_is_real",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-023",
+ "input": "We have enough funding to operate for 18 months.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "funding_period",
+ "found": false
+ },
+ {
+ "concept": "operational_sustainability",
+ "found": false
+ },
+ {
+ "concept": "burn_rate_unknown",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "no_risk_exists",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-024",
+ "input": "The new feature was deployed at 3am and user complaints tripled the next day.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "feature_deployment",
+ "found": false
+ },
+ {
+ "concept": "timing_3am",
+ "found": false
+ },
+ {
+ "concept": "complaint_tripling",
+ "found": false
+ },
+ {
+ "concept": "temporal_relationship",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "deployment_caused_complaints",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-025",
+ "input": "We need to launch a mobile app to capture market share.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "mobile_app_proposed",
+ "found": false
+ },
+ {
+ "concept": "market_share_desired",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "no_mobile_app_exists",
+ "absent": true
+ },
+ {
+ "concept": "competitors_have_apps",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-026",
+ "input": "The system has been running for 90 days without failure since the migration.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "uptime_90_days",
+ "found": false
+ },
+ {
+ "concept": "post_migration_context",
+ "found": false
+ },
+ {
+ "concept": "baseline_established",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "system_is_stable_forever",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-027",
+ "input": "No one has submitted the required compliance report despite multiple reminders.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "compliance_report",
+ "found": false
+ },
+ {
+ "concept": "multiple_reminders",
+ "found": false
+ },
+ {
+ "concept": "non_submission",
+ "found": false
+ },
+ {
+ "concept": "absent_action",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "deliberate_refusal",
+ "absent": true
+ },
+ {
+ "concept": "negligence",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-028",
+ "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "audit_findings",
+ "found": false
+ },
+ {
+ "concept": "incorrect_reports_rate_3_of_10",
+ "found": false
+ },
+ {
+ "concept": "data_accuracy_issue",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "intentional_falsification",
+ "absent": true
+ },
+ {
+ "concept": "systemic_failure",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-029",
+ "input": "We should implement the new CRM because our competitors have one.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "crm_proposal",
+ "found": false
+ },
+ {
+ "concept": "competitor_comparison",
+ "found": false
+ },
+ {
+ "concept": "competitive_pressure",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "crm_will_help",
+ "absent": true
+ },
+ {
+ "concept": "we_lack_crm",
+ "absent": true
+ },
+ {
+ "concept": "competitors_success_is_from_crm",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-030",
+ "input": "The server response time was acceptable last quarter but degraded this month.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "response_time_baseline_acceptable",
+ "found": false
+ },
+ {
+ "concept": "degradation_timeline",
+ "found": false
+ },
+ {
+ "concept": "quarter_to_month_comparison",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "load_increase_occurred",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-031",
+ "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "regulatory_requirement",
+ "found": false
+ },
+ {
+ "concept": "data_location_violation",
+ "found": false
+ },
+ {
+ "concept": "cross_border_backup",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "compliance_failure_is_certain",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-032",
+ "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "industry_decline_prediction",
+ "found": false
+ },
+ {
+ "concept": "external_source",
+ "found": false
+ },
+ {
+ "concept": "regulatory_cause",
+ "found": false
+ },
+ {
+ "concept": "15_percent_forecast",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "decline_will_occur",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-033",
+ "input": "The database schema was changed on Friday but the reports are still working.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "schema_change",
+ "found": false
+ },
+ {
+ "concept": "reports_working_post_change",
+ "found": false
+ },
+ {
+ "concept": "unexpected_continuity",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "change_was_harmless",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-034",
+ "input": "Some team members say the new process is better while others say it's slower.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "subjective_split",
+ "found": false
+ },
+ {
+ "concept": "new_process_evaluation",
+ "found": false
+ },
+ {
+ "concept": "conflicting_opinions",
+ "found": false
+ },
+ {
+ "concept": "measurement_gap",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "process_is_better_or_worse",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-035",
+ "input": "The application works fine on Chrome but not on Safari.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "chrome_compatibility",
+ "found": false
+ },
+ {
+ "concept": "safari_incompatibility",
+ "found": false
+ },
+ {
+ "concept": "browser_specific_issue",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "webkit_bug",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests-results/evaluation-2026-08-01T05-22-52.json b/tests-results/evaluation-2026-08-01T05-22-52.json
new file mode 100644
index 0000000..f5a1685
--- /dev/null
+++ b/tests-results/evaluation-2026-08-01T05-22-52.json
@@ -0,0 +1,1735 @@
+{
+ "timestamp": "2026-08-01T05:22:52.077Z",
+ "provider": "mock",
+ "promptVersion": "v0.2",
+ "casesRun": 35,
+ "summary": {
+ "technical": {
+ "schemaValidityRate": "100.0%",
+ "classificationMatchRate": "100.0%",
+ "nextQuestionPresentRate": "100.0%",
+ "passRate": "100.0%"
+ },
+ "reasoningQuality": {
+ "requiredConceptMatchRate": "0.0%",
+ "unsupportedInferenceFailures": "0",
+ "passRate": "0.0%"
+ },
+ "combinedPassRate": "0.0%",
+ "averageResponseDurationMs": "1"
+ },
+ "testCaseResults": [
+ {
+ "id": "tc-001",
+ "input": "All customers cannot download their invoices.",
+ "responseDurationMs": 4,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "customers",
+ "found": false
+ },
+ {
+ "concept": "invoices",
+ "found": false
+ },
+ {
+ "concept": "download",
+ "found": false
+ },
+ {
+ "concept": "access_issue",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-002",
+ "input": "Some customers cannot download their invoices.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "some_customers",
+ "found": false
+ },
+ {
+ "concept": "invoices",
+ "found": false
+ },
+ {
+ "concept": "download",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "root_cause",
+ "absent": true
+ },
+ {
+ "concept": "payment_system_failure",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-003",
+ "input": "Complaints increased by 35%.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "complaints",
+ "found": true
+ },
+ {
+ "concept": "increase",
+ "found": false
+ },
+ {
+ "concept": "35_percent",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cause_of_complaints",
+ "absent": true
+ },
+ {
+ "concept": "customer_dissatisfaction_is_worse",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-004",
+ "input": "Complaints increased by 35% while production increased by 40%.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "complaints_increase",
+ "found": false
+ },
+ {
+ "concept": "production_increase",
+ "found": false
+ },
+ {
+ "concept": "relative_rates",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "production_quality_declined",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-005",
+ "input": "Sales are falling.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "sales_decline",
+ "found": false
+ },
+ {
+ "concept": "direction_negative",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cause_of_fall",
+ "absent": true
+ },
+ {
+ "concept": "competitor_action",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-006",
+ "input": "Sales fell sharply immediately after the price increase.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "sales_decline",
+ "found": false
+ },
+ {
+ "concept": "price_increase",
+ "found": false
+ },
+ {
+ "concept": "temporal_correlation",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "price_increase_caused_the_fall",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-007",
+ "input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "investigate_contradiction",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "revenue_above_target",
+ "found": false
+ },
+ {
+ "concept": "profit_decline",
+ "found": false
+ },
+ {
+ "concept": "divergence",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cost_overrun_is_the_cause",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-008",
+ "input": "Revenue from the premium tier dropped while total revenue grew.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "premium_tier_decline",
+ "found": false
+ },
+ {
+ "concept": "total_revenue_growth",
+ "found": false
+ },
+ {
+ "concept": "segment_cannibalization_risk",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "pricing_change_occurred",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-009",
+ "input": "We need to improve our customer retention rate.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "retention_improvement_desired",
+ "found": false
+ },
+ {
+ "concept": "current_state_unknown",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "retention_rate_is_low",
+ "absent": true
+ },
+ {
+ "concept": "churn_has_increased",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-010",
+ "input": "The system latency went from 200ms to 5 seconds on Tuesday.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "latency_baseline_200ms",
+ "found": false
+ },
+ {
+ "concept": "latency_spike_5s",
+ "found": false
+ },
+ {
+ "concept": "timestamp_tuesday",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "database_cause",
+ "absent": true
+ },
+ {
+ "concept": "release_cause",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-011",
+ "input": "The new release should fix the login issue.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "proposed_solution",
+ "found": false
+ },
+ {
+ "concept": "login_issue",
+ "found": false
+ },
+ {
+ "concept": "solution_claim",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "login_issue_is_real",
+ "absent": true
+ },
+ {
+ "concept": "release_will_work",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-012",
+ "input": "I think therefore I am.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "philosophical_statement",
+ "found": false
+ },
+ {
+ "concept": "insufficient_operational_context",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "business_problem_exists",
+ "absent": true
+ },
+ {
+ "concept": "actionable_insight_possible",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-013",
+ "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "meta_context",
+ "found": false
+ },
+ {
+ "concept": "testing_hypothesis",
+ "found": false
+ },
+ {
+ "concept": "self_reference",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-014",
+ "input": "The warehouse manager reported that inventory counts don't match the system.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "warehouse_manager_report",
+ "found": false
+ },
+ {
+ "concept": "inventory_mismatch",
+ "found": false
+ },
+ {
+ "concept": "system_discrepancy",
+ "found": false
+ },
+ {
+ "concept": "source_attribution",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "theft_occurred",
+ "absent": true
+ },
+ {
+ "concept": "software_bug",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-015",
+ "input": "We've seen a 35% increase in customer complaints.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "complaints_increase",
+ "found": false
+ },
+ {
+ "concept": "percentage_metric",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "product_quality_declined",
+ "absent": true
+ },
+ {
+ "concept": "customer_satisfaction_drop",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-016",
+ "input": "The number of active users increased by 500%, from 4 to 2,001.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "active_users_metric",
+ "found": false
+ },
+ {
+ "concept": "absolute_vs_relative_growth",
+ "found": false
+ },
+ {
+ "concept": "small_base_problem",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "product_success",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-017",
+ "input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "investigate_contradiction",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "engagement_improvement",
+ "found": false
+ },
+ {
+ "concept": "support_backlog_growth",
+ "found": false
+ },
+ {
+ "concept": "divergent_metrics",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "users_are_angry",
+ "absent": true
+ },
+ {
+ "concept": "product_quality_is_worse",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-018",
+ "input": "The manufacturing team needs better quality control.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "manufacturing_team",
+ "found": false
+ },
+ {
+ "concept": "quality_control_desired",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "quality_is_bad",
+ "absent": true
+ },
+ {
+ "concept": "defect_rate_is_high",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-019",
+ "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "eu_region",
+ "found": false
+ },
+ {
+ "concept": "403_error",
+ "found": false
+ },
+ {
+ "concept": "access_denied",
+ "found": false
+ },
+ {
+ "concept": "geographic_scope",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "gdpr_cause",
+ "absent": true
+ },
+ {
+ "concept": "regulatory_change",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-020",
+ "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "eu_region_subset",
+ "found": false
+ },
+ {
+ "concept": "403_error",
+ "found": false
+ },
+ {
+ "concept": "partial_reachability",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "all_eu_users_affected",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-021",
+ "input": "Production output was 1,200 units last month and 1,180 units this month.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "production_output",
+ "found": false
+ },
+ {
+ "concept": "month_over_month_decline",
+ "found": false
+ },
+ {
+ "concept": "absolute_difference",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "efficiency_loss_occurred",
+ "absent": true
+ },
+ {
+ "concept": "equipment_failure",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-022",
+ "input": "The CFO reported that the company's cash position is healthy.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "cfo_statement",
+ "found": false
+ },
+ {
+ "concept": "cash_position_claim",
+ "found": false
+ },
+ {
+ "concept": "source_attribution_cfo",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cash_is_healthy",
+ "absent": true
+ },
+ {
+ "concept": "financial_stability_is_real",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-023",
+ "input": "We have enough funding to operate for 18 months.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "funding_period",
+ "found": false
+ },
+ {
+ "concept": "operational_sustainability",
+ "found": false
+ },
+ {
+ "concept": "burn_rate_unknown",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "no_risk_exists",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-024",
+ "input": "The new feature was deployed at 3am and user complaints tripled the next day.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "feature_deployment",
+ "found": false
+ },
+ {
+ "concept": "timing_3am",
+ "found": false
+ },
+ {
+ "concept": "complaint_tripling",
+ "found": false
+ },
+ {
+ "concept": "temporal_relationship",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "deployment_caused_complaints",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-025",
+ "input": "We need to launch a mobile app to capture market share.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "mobile_app_proposed",
+ "found": false
+ },
+ {
+ "concept": "market_share_desired",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "no_mobile_app_exists",
+ "absent": true
+ },
+ {
+ "concept": "competitors_have_apps",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-026",
+ "input": "The system has been running for 90 days without failure since the migration.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "uptime_90_days",
+ "found": false
+ },
+ {
+ "concept": "post_migration_context",
+ "found": false
+ },
+ {
+ "concept": "baseline_established",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "system_is_stable_forever",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-027",
+ "input": "No one has submitted the required compliance report despite multiple reminders.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "compliance_report",
+ "found": false
+ },
+ {
+ "concept": "multiple_reminders",
+ "found": false
+ },
+ {
+ "concept": "non_submission",
+ "found": false
+ },
+ {
+ "concept": "absent_action",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "deliberate_refusal",
+ "absent": true
+ },
+ {
+ "concept": "negligence",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-028",
+ "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "audit_findings",
+ "found": false
+ },
+ {
+ "concept": "incorrect_reports_rate_3_of_10",
+ "found": false
+ },
+ {
+ "concept": "data_accuracy_issue",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "intentional_falsification",
+ "absent": true
+ },
+ {
+ "concept": "systemic_failure",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-029",
+ "input": "We should implement the new CRM because our competitors have one.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "crm_proposal",
+ "found": false
+ },
+ {
+ "concept": "competitor_comparison",
+ "found": false
+ },
+ {
+ "concept": "competitive_pressure",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "crm_will_help",
+ "absent": true
+ },
+ {
+ "concept": "we_lack_crm",
+ "absent": true
+ },
+ {
+ "concept": "competitors_success_is_from_crm",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-030",
+ "input": "The server response time was acceptable last quarter but degraded this month.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "response_time_baseline_acceptable",
+ "found": false
+ },
+ {
+ "concept": "degradation_timeline",
+ "found": false
+ },
+ {
+ "concept": "quarter_to_month_comparison",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "load_increase_occurred",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-031",
+ "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "regulatory_requirement",
+ "found": false
+ },
+ {
+ "concept": "data_location_violation",
+ "found": false
+ },
+ {
+ "concept": "cross_border_backup",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "compliance_failure_is_certain",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-032",
+ "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "industry_decline_prediction",
+ "found": false
+ },
+ {
+ "concept": "external_source",
+ "found": false
+ },
+ {
+ "concept": "regulatory_cause",
+ "found": false
+ },
+ {
+ "concept": "15_percent_forecast",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "decline_will_occur",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-033",
+ "input": "The database schema was changed on Friday but the reports are still working.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "schema_change",
+ "found": false
+ },
+ {
+ "concept": "reports_working_post_change",
+ "found": false
+ },
+ {
+ "concept": "unexpected_continuity",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "change_was_harmless",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-034",
+ "input": "Some team members say the new process is better while others say it's slower.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "subjective_split",
+ "found": false
+ },
+ {
+ "concept": "new_process_evaluation",
+ "found": false
+ },
+ {
+ "concept": "conflicting_opinions",
+ "found": false
+ },
+ {
+ "concept": "measurement_gap",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "process_is_better_or_worse",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-035",
+ "input": "The application works fine on Chrome but not on Safari.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": null,
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "chrome_compatibility",
+ "found": false
+ },
+ {
+ "concept": "safari_incompatibility",
+ "found": false
+ },
+ {
+ "concept": "browser_specific_issue",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "webkit_bug",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests-results/evaluation-2026-08-01T05-30-52.json b/tests-results/evaluation-2026-08-01T05-30-52.json
new file mode 100644
index 0000000..8d32c0f
--- /dev/null
+++ b/tests-results/evaluation-2026-08-01T05-30-52.json
@@ -0,0 +1,1735 @@
+{
+ "timestamp": "2026-08-01T05:30:52.558Z",
+ "provider": "mock",
+ "promptVersion": "v0.2",
+ "casesRun": 35,
+ "summary": {
+ "technical": {
+ "schemaValidityRate": "100.0%",
+ "classificationMatchRate": "100.0%",
+ "nextQuestionPresentRate": "100.0%",
+ "passRate": "100.0%"
+ },
+ "reasoningQuality": {
+ "requiredConceptMatchRate": "0.0%",
+ "unsupportedInferenceFailures": "0",
+ "passRate": "0.0%"
+ },
+ "combinedPassRate": "0.0%",
+ "averageResponseDurationMs": "1"
+ },
+ "testCaseResults": [
+ {
+ "id": "tc-001",
+ "input": "All customers cannot download their invoices.",
+ "responseDurationMs": 4,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "customers",
+ "found": false
+ },
+ {
+ "concept": "invoices",
+ "found": false
+ },
+ {
+ "concept": "download",
+ "found": false
+ },
+ {
+ "concept": "access_issue",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-002",
+ "input": "Some customers cannot download their invoices.",
+ "responseDurationMs": 2,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "some_customers",
+ "found": false
+ },
+ {
+ "concept": "invoices",
+ "found": false
+ },
+ {
+ "concept": "download",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "root_cause",
+ "absent": true
+ },
+ {
+ "concept": "payment_system_failure",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-003",
+ "input": "Complaints increased by 35%.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "complaints",
+ "found": true
+ },
+ {
+ "concept": "increase",
+ "found": false
+ },
+ {
+ "concept": "35_percent",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cause_of_complaints",
+ "absent": true
+ },
+ {
+ "concept": "customer_dissatisfaction_is_worse",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-004",
+ "input": "Complaints increased by 35% while production increased by 40%.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "complaints_increase",
+ "found": false
+ },
+ {
+ "concept": "production_increase",
+ "found": false
+ },
+ {
+ "concept": "relative_rates",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "production_quality_declined",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-005",
+ "input": "Sales are falling.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "sales_decline",
+ "found": false
+ },
+ {
+ "concept": "direction_negative",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cause_of_fall",
+ "absent": true
+ },
+ {
+ "concept": "competitor_action",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-006",
+ "input": "Sales fell sharply immediately after the price increase.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "sales_decline",
+ "found": false
+ },
+ {
+ "concept": "price_increase",
+ "found": false
+ },
+ {
+ "concept": "temporal_correlation",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "price_increase_caused_the_fall",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-007",
+ "input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "investigate_contradiction",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "revenue_above_target",
+ "found": false
+ },
+ {
+ "concept": "profit_decline",
+ "found": false
+ },
+ {
+ "concept": "divergence",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cost_overrun_is_the_cause",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-008",
+ "input": "Revenue from the premium tier dropped while total revenue grew.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "premium_tier_decline",
+ "found": false
+ },
+ {
+ "concept": "total_revenue_growth",
+ "found": false
+ },
+ {
+ "concept": "segment_cannibalization_risk",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "pricing_change_occurred",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-009",
+ "input": "We need to improve our customer retention rate.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "retention_improvement_desired",
+ "found": false
+ },
+ {
+ "concept": "current_state_unknown",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "retention_rate_is_low",
+ "absent": true
+ },
+ {
+ "concept": "churn_has_increased",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-010",
+ "input": "The system latency went from 200ms to 5 seconds on Tuesday.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "latency_baseline_200ms",
+ "found": false
+ },
+ {
+ "concept": "latency_spike_5s",
+ "found": false
+ },
+ {
+ "concept": "timestamp_tuesday",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "database_cause",
+ "absent": true
+ },
+ {
+ "concept": "release_cause",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-011",
+ "input": "The new release should fix the login issue.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "proposed_solution",
+ "found": false
+ },
+ {
+ "concept": "login_issue",
+ "found": false
+ },
+ {
+ "concept": "solution_claim",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "login_issue_is_real",
+ "absent": true
+ },
+ {
+ "concept": "release_will_work",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-012",
+ "input": "I think therefore I am.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "ambiguous_statement",
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "philosophical_statement",
+ "found": false
+ },
+ {
+ "concept": "insufficient_operational_context",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "business_problem_exists",
+ "absent": true
+ },
+ {
+ "concept": "actionable_insight_possible",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-013",
+ "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "ambiguous_statement",
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "meta_context",
+ "found": false
+ },
+ {
+ "concept": "testing_hypothesis",
+ "found": false
+ },
+ {
+ "concept": "self_reference",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-014",
+ "input": "The warehouse manager reported that inventory counts don't match the system.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "warehouse_manager_report",
+ "found": false
+ },
+ {
+ "concept": "inventory_mismatch",
+ "found": false
+ },
+ {
+ "concept": "system_discrepancy",
+ "found": false
+ },
+ {
+ "concept": "source_attribution",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "theft_occurred",
+ "absent": true
+ },
+ {
+ "concept": "software_bug",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-015",
+ "input": "We've seen a 35% increase in customer complaints.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "complaints_increase",
+ "found": false
+ },
+ {
+ "concept": "percentage_metric",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "product_quality_declined",
+ "absent": true
+ },
+ {
+ "concept": "customer_satisfaction_drop",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-016",
+ "input": "The number of active users increased by 500%, from 4 to 2,001.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "active_users_metric",
+ "found": false
+ },
+ {
+ "concept": "absolute_vs_relative_growth",
+ "found": false
+ },
+ {
+ "concept": "small_base_problem",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "product_success",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-017",
+ "input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "investigate_contradiction",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "engagement_improvement",
+ "found": false
+ },
+ {
+ "concept": "support_backlog_growth",
+ "found": false
+ },
+ {
+ "concept": "divergent_metrics",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "users_are_angry",
+ "absent": true
+ },
+ {
+ "concept": "product_quality_is_worse",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-018",
+ "input": "The manufacturing team needs better quality control.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "manufacturing_team",
+ "found": false
+ },
+ {
+ "concept": "quality_control_desired",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "quality_is_bad",
+ "absent": true
+ },
+ {
+ "concept": "defect_rate_is_high",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-019",
+ "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "eu_region",
+ "found": false
+ },
+ {
+ "concept": "403_error",
+ "found": false
+ },
+ {
+ "concept": "access_denied",
+ "found": false
+ },
+ {
+ "concept": "geographic_scope",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "gdpr_cause",
+ "absent": true
+ },
+ {
+ "concept": "regulatory_change",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-020",
+ "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "eu_region_subset",
+ "found": false
+ },
+ {
+ "concept": "403_error",
+ "found": false
+ },
+ {
+ "concept": "partial_reachability",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "all_eu_users_affected",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-021",
+ "input": "Production output was 1,200 units last month and 1,180 units this month.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "production_output",
+ "found": false
+ },
+ {
+ "concept": "month_over_month_decline",
+ "found": false
+ },
+ {
+ "concept": "absolute_difference",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "efficiency_loss_occurred",
+ "absent": true
+ },
+ {
+ "concept": "equipment_failure",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-022",
+ "input": "The CFO reported that the company's cash position is healthy.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "cfo_statement",
+ "found": false
+ },
+ {
+ "concept": "cash_position_claim",
+ "found": false
+ },
+ {
+ "concept": "source_attribution_cfo",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cash_is_healthy",
+ "absent": true
+ },
+ {
+ "concept": "financial_stability_is_real",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-023",
+ "input": "We have enough funding to operate for 18 months.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "funding_period",
+ "found": false
+ },
+ {
+ "concept": "operational_sustainability",
+ "found": false
+ },
+ {
+ "concept": "burn_rate_unknown",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "no_risk_exists",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-024",
+ "input": "The new feature was deployed at 3am and user complaints tripled the next day.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "feature_deployment",
+ "found": false
+ },
+ {
+ "concept": "timing_3am",
+ "found": false
+ },
+ {
+ "concept": "complaint_tripling",
+ "found": false
+ },
+ {
+ "concept": "temporal_relationship",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "deployment_caused_complaints",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-025",
+ "input": "We need to launch a mobile app to capture market share.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "mobile_app_proposed",
+ "found": false
+ },
+ {
+ "concept": "market_share_desired",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "no_mobile_app_exists",
+ "absent": true
+ },
+ {
+ "concept": "competitors_have_apps",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-026",
+ "input": "The system has been running for 90 days without failure since the migration.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "uptime_90_days",
+ "found": false
+ },
+ {
+ "concept": "post_migration_context",
+ "found": false
+ },
+ {
+ "concept": "baseline_established",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "system_is_stable_forever",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-027",
+ "input": "No one has submitted the required compliance report despite multiple reminders.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "compliance_report",
+ "found": false
+ },
+ {
+ "concept": "multiple_reminders",
+ "found": false
+ },
+ {
+ "concept": "non_submission",
+ "found": false
+ },
+ {
+ "concept": "absent_action",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "deliberate_refusal",
+ "absent": true
+ },
+ {
+ "concept": "negligence",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-028",
+ "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "audit_findings",
+ "found": false
+ },
+ {
+ "concept": "incorrect_reports_rate_3_of_10",
+ "found": false
+ },
+ {
+ "concept": "data_accuracy_issue",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "intentional_falsification",
+ "absent": true
+ },
+ {
+ "concept": "systemic_failure",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-029",
+ "input": "We should implement the new CRM because our competitors have one.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "crm_proposal",
+ "found": false
+ },
+ {
+ "concept": "competitor_comparison",
+ "found": false
+ },
+ {
+ "concept": "competitive_pressure",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "crm_will_help",
+ "absent": true
+ },
+ {
+ "concept": "we_lack_crm",
+ "absent": true
+ },
+ {
+ "concept": "competitors_success_is_from_crm",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-030",
+ "input": "The server response time was acceptable last quarter but degraded this month.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "response_time_baseline_acceptable",
+ "found": false
+ },
+ {
+ "concept": "degradation_timeline",
+ "found": false
+ },
+ {
+ "concept": "quarter_to_month_comparison",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "load_increase_occurred",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-031",
+ "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "regulatory_requirement",
+ "found": false
+ },
+ {
+ "concept": "data_location_violation",
+ "found": false
+ },
+ {
+ "concept": "cross_border_backup",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "compliance_failure_is_certain",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-032",
+ "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "industry_decline_prediction",
+ "found": false
+ },
+ {
+ "concept": "external_source",
+ "found": false
+ },
+ {
+ "concept": "regulatory_cause",
+ "found": false
+ },
+ {
+ "concept": "15_percent_forecast",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "decline_will_occur",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-033",
+ "input": "The database schema was changed on Friday but the reports are still working.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "schema_change",
+ "found": false
+ },
+ {
+ "concept": "reports_working_post_change",
+ "found": false
+ },
+ {
+ "concept": "unexpected_continuity",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "change_was_harmless",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-034",
+ "input": "Some team members say the new process is better while others say it's slower.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "subjective_split",
+ "found": false
+ },
+ {
+ "concept": "new_process_evaluation",
+ "found": false
+ },
+ {
+ "concept": "conflicting_opinions",
+ "found": false
+ },
+ {
+ "concept": "measurement_gap",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "process_is_better_or_worse",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-035",
+ "input": "The application works fine on Chrome but not on Safari.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "chrome_compatibility",
+ "found": false
+ },
+ {
+ "concept": "safari_incompatibility",
+ "found": false
+ },
+ {
+ "concept": "browser_specific_issue",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "webkit_bug",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests-results/evaluation-2026-08-01T05-31-49.json b/tests-results/evaluation-2026-08-01T05-31-49.json
new file mode 100644
index 0000000..ab319e0
--- /dev/null
+++ b/tests-results/evaluation-2026-08-01T05-31-49.json
@@ -0,0 +1,1735 @@
+{
+ "timestamp": "2026-08-01T05:31:49.869Z",
+ "provider": "mock",
+ "promptVersion": "v0.2",
+ "casesRun": 35,
+ "summary": {
+ "technical": {
+ "schemaValidityRate": "100.0%",
+ "classificationMatchRate": "100.0%",
+ "nextQuestionPresentRate": "100.0%",
+ "passRate": "100.0%"
+ },
+ "reasoningQuality": {
+ "requiredConceptMatchRate": "0.0%",
+ "unsupportedInferenceFailures": "0",
+ "passRate": "0.0%"
+ },
+ "combinedPassRate": "0.0%",
+ "averageResponseDurationMs": "1"
+ },
+ "testCaseResults": [
+ {
+ "id": "tc-001",
+ "input": "All customers cannot download their invoices.",
+ "responseDurationMs": 5,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "customers",
+ "found": false
+ },
+ {
+ "concept": "invoices",
+ "found": false
+ },
+ {
+ "concept": "download",
+ "found": false
+ },
+ {
+ "concept": "access_issue",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-002",
+ "input": "Some customers cannot download their invoices.",
+ "responseDurationMs": 2,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "some_customers",
+ "found": false
+ },
+ {
+ "concept": "invoices",
+ "found": false
+ },
+ {
+ "concept": "download",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "root_cause",
+ "absent": true
+ },
+ {
+ "concept": "payment_system_failure",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-003",
+ "input": "Complaints increased by 35%.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "complaints",
+ "found": true
+ },
+ {
+ "concept": "increase",
+ "found": false
+ },
+ {
+ "concept": "35_percent",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cause_of_complaints",
+ "absent": true
+ },
+ {
+ "concept": "customer_dissatisfaction_is_worse",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-004",
+ "input": "Complaints increased by 35% while production increased by 40%.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "complaints_increase",
+ "found": false
+ },
+ {
+ "concept": "production_increase",
+ "found": false
+ },
+ {
+ "concept": "relative_rates",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "production_quality_declined",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-005",
+ "input": "Sales are falling.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "sales_decline",
+ "found": false
+ },
+ {
+ "concept": "direction_negative",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cause_of_fall",
+ "absent": true
+ },
+ {
+ "concept": "competitor_action",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-006",
+ "input": "Sales fell sharply immediately after the price increase.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "sales_decline",
+ "found": false
+ },
+ {
+ "concept": "price_increase",
+ "found": false
+ },
+ {
+ "concept": "temporal_correlation",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "price_increase_caused_the_fall",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-007",
+ "input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "investigate_contradiction",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "revenue_above_target",
+ "found": false
+ },
+ {
+ "concept": "profit_decline",
+ "found": false
+ },
+ {
+ "concept": "divergence",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cost_overrun_is_the_cause",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-008",
+ "input": "Revenue from the premium tier dropped while total revenue grew.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "premium_tier_decline",
+ "found": false
+ },
+ {
+ "concept": "total_revenue_growth",
+ "found": false
+ },
+ {
+ "concept": "segment_cannibalization_risk",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "pricing_change_occurred",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-009",
+ "input": "We need to improve our customer retention rate.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "retention_improvement_desired",
+ "found": false
+ },
+ {
+ "concept": "current_state_unknown",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "retention_rate_is_low",
+ "absent": true
+ },
+ {
+ "concept": "churn_has_increased",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-010",
+ "input": "The system latency went from 200ms to 5 seconds on Tuesday.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "latency_baseline_200ms",
+ "found": false
+ },
+ {
+ "concept": "latency_spike_5s",
+ "found": false
+ },
+ {
+ "concept": "timestamp_tuesday",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "database_cause",
+ "absent": true
+ },
+ {
+ "concept": "release_cause",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-011",
+ "input": "The new release should fix the login issue.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "proposed_solution",
+ "found": false
+ },
+ {
+ "concept": "login_issue",
+ "found": false
+ },
+ {
+ "concept": "solution_claim",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "login_issue_is_real",
+ "absent": true
+ },
+ {
+ "concept": "release_will_work",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-012",
+ "input": "I think therefore I am.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "ambiguous_statement",
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "philosophical_statement",
+ "found": false
+ },
+ {
+ "concept": "insufficient_operational_context",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "business_problem_exists",
+ "absent": true
+ },
+ {
+ "concept": "actionable_insight_possible",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-013",
+ "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "ambiguous_statement",
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "meta_context",
+ "found": false
+ },
+ {
+ "concept": "testing_hypothesis",
+ "found": false
+ },
+ {
+ "concept": "self_reference",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-014",
+ "input": "The warehouse manager reported that inventory counts don't match the system.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "warehouse_manager_report",
+ "found": false
+ },
+ {
+ "concept": "inventory_mismatch",
+ "found": false
+ },
+ {
+ "concept": "system_discrepancy",
+ "found": false
+ },
+ {
+ "concept": "source_attribution",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "theft_occurred",
+ "absent": true
+ },
+ {
+ "concept": "software_bug",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-015",
+ "input": "We've seen a 35% increase in customer complaints.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "complaints_increase",
+ "found": false
+ },
+ {
+ "concept": "percentage_metric",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "product_quality_declined",
+ "absent": true
+ },
+ {
+ "concept": "customer_satisfaction_drop",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-016",
+ "input": "The number of active users increased by 500%, from 4 to 2,001.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "active_users_metric",
+ "found": false
+ },
+ {
+ "concept": "absolute_vs_relative_growth",
+ "found": false
+ },
+ {
+ "concept": "small_base_problem",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "product_success",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-017",
+ "input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "investigate_contradiction",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "engagement_improvement",
+ "found": false
+ },
+ {
+ "concept": "support_backlog_growth",
+ "found": false
+ },
+ {
+ "concept": "divergent_metrics",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "users_are_angry",
+ "absent": true
+ },
+ {
+ "concept": "product_quality_is_worse",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-018",
+ "input": "The manufacturing team needs better quality control.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "manufacturing_team",
+ "found": false
+ },
+ {
+ "concept": "quality_control_desired",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "quality_is_bad",
+ "absent": true
+ },
+ {
+ "concept": "defect_rate_is_high",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-019",
+ "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "eu_region",
+ "found": false
+ },
+ {
+ "concept": "403_error",
+ "found": false
+ },
+ {
+ "concept": "access_denied",
+ "found": false
+ },
+ {
+ "concept": "geographic_scope",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "gdpr_cause",
+ "absent": true
+ },
+ {
+ "concept": "regulatory_change",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-020",
+ "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "eu_region_subset",
+ "found": false
+ },
+ {
+ "concept": "403_error",
+ "found": false
+ },
+ {
+ "concept": "partial_reachability",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "all_eu_users_affected",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-021",
+ "input": "Production output was 1,200 units last month and 1,180 units this month.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "production_output",
+ "found": false
+ },
+ {
+ "concept": "month_over_month_decline",
+ "found": false
+ },
+ {
+ "concept": "absolute_difference",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "efficiency_loss_occurred",
+ "absent": true
+ },
+ {
+ "concept": "equipment_failure",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-022",
+ "input": "The CFO reported that the company's cash position is healthy.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "cfo_statement",
+ "found": false
+ },
+ {
+ "concept": "cash_position_claim",
+ "found": false
+ },
+ {
+ "concept": "source_attribution_cfo",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cash_is_healthy",
+ "absent": true
+ },
+ {
+ "concept": "financial_stability_is_real",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-023",
+ "input": "We have enough funding to operate for 18 months.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "funding_period",
+ "found": false
+ },
+ {
+ "concept": "operational_sustainability",
+ "found": false
+ },
+ {
+ "concept": "burn_rate_unknown",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "no_risk_exists",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-024",
+ "input": "The new feature was deployed at 3am and user complaints tripled the next day.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "feature_deployment",
+ "found": false
+ },
+ {
+ "concept": "timing_3am",
+ "found": false
+ },
+ {
+ "concept": "complaint_tripling",
+ "found": false
+ },
+ {
+ "concept": "temporal_relationship",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "deployment_caused_complaints",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-025",
+ "input": "We need to launch a mobile app to capture market share.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "mobile_app_proposed",
+ "found": false
+ },
+ {
+ "concept": "market_share_desired",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "no_mobile_app_exists",
+ "absent": true
+ },
+ {
+ "concept": "competitors_have_apps",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-026",
+ "input": "The system has been running for 90 days without failure since the migration.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "uptime_90_days",
+ "found": false
+ },
+ {
+ "concept": "post_migration_context",
+ "found": false
+ },
+ {
+ "concept": "baseline_established",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "system_is_stable_forever",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-027",
+ "input": "No one has submitted the required compliance report despite multiple reminders.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "compliance_report",
+ "found": false
+ },
+ {
+ "concept": "multiple_reminders",
+ "found": false
+ },
+ {
+ "concept": "non_submission",
+ "found": false
+ },
+ {
+ "concept": "absent_action",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "deliberate_refusal",
+ "absent": true
+ },
+ {
+ "concept": "negligence",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-028",
+ "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "audit_findings",
+ "found": false
+ },
+ {
+ "concept": "incorrect_reports_rate_3_of_10",
+ "found": false
+ },
+ {
+ "concept": "data_accuracy_issue",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "intentional_falsification",
+ "absent": true
+ },
+ {
+ "concept": "systemic_failure",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-029",
+ "input": "We should implement the new CRM because our competitors have one.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "crm_proposal",
+ "found": false
+ },
+ {
+ "concept": "competitor_comparison",
+ "found": false
+ },
+ {
+ "concept": "competitive_pressure",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "crm_will_help",
+ "absent": true
+ },
+ {
+ "concept": "we_lack_crm",
+ "absent": true
+ },
+ {
+ "concept": "competitors_success_is_from_crm",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-030",
+ "input": "The server response time was acceptable last quarter but degraded this month.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "response_time_baseline_acceptable",
+ "found": false
+ },
+ {
+ "concept": "degradation_timeline",
+ "found": false
+ },
+ {
+ "concept": "quarter_to_month_comparison",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "load_increase_occurred",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-031",
+ "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "regulatory_requirement",
+ "found": false
+ },
+ {
+ "concept": "data_location_violation",
+ "found": false
+ },
+ {
+ "concept": "cross_border_backup",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "compliance_failure_is_certain",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-032",
+ "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "industry_decline_prediction",
+ "found": false
+ },
+ {
+ "concept": "external_source",
+ "found": false
+ },
+ {
+ "concept": "regulatory_cause",
+ "found": false
+ },
+ {
+ "concept": "15_percent_forecast",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "decline_will_occur",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-033",
+ "input": "The database schema was changed on Friday but the reports are still working.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "schema_change",
+ "found": false
+ },
+ {
+ "concept": "reports_working_post_change",
+ "found": false
+ },
+ {
+ "concept": "unexpected_continuity",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "change_was_harmless",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-034",
+ "input": "Some team members say the new process is better while others say it's slower.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "subjective_split",
+ "found": false
+ },
+ {
+ "concept": "new_process_evaluation",
+ "found": false
+ },
+ {
+ "concept": "conflicting_opinions",
+ "found": false
+ },
+ {
+ "concept": "measurement_gap",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "process_is_better_or_worse",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-035",
+ "input": "The application works fine on Chrome but not on Safari.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "chrome_compatibility",
+ "found": false
+ },
+ {
+ "concept": "safari_incompatibility",
+ "found": false
+ },
+ {
+ "concept": "browser_specific_issue",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "webkit_bug",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests-results/evaluation-2026-08-01T06-02-26.json b/tests-results/evaluation-2026-08-01T06-02-26.json
new file mode 100644
index 0000000..62b6268
--- /dev/null
+++ b/tests-results/evaluation-2026-08-01T06-02-26.json
@@ -0,0 +1,1735 @@
+{
+ "timestamp": "2026-08-01T06:02:26.238Z",
+ "provider": "mock",
+ "promptVersion": "v0.2",
+ "casesRun": 35,
+ "summary": {
+ "technical": {
+ "schemaValidityRate": "100.0%",
+ "classificationMatchRate": "100.0%",
+ "nextQuestionPresentRate": "100.0%",
+ "passRate": "100.0%"
+ },
+ "reasoningQuality": {
+ "requiredConceptMatchRate": "0.0%",
+ "unsupportedInferenceFailures": "0",
+ "passRate": "0.0%"
+ },
+ "combinedPassRate": "0.0%",
+ "averageResponseDurationMs": "1"
+ },
+ "testCaseResults": [
+ {
+ "id": "tc-001",
+ "input": "All customers cannot download their invoices.",
+ "responseDurationMs": 4,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "customers",
+ "found": false
+ },
+ {
+ "concept": "invoices",
+ "found": false
+ },
+ {
+ "concept": "download",
+ "found": false
+ },
+ {
+ "concept": "access_issue",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-002",
+ "input": "Some customers cannot download their invoices.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "some_customers",
+ "found": false
+ },
+ {
+ "concept": "invoices",
+ "found": false
+ },
+ {
+ "concept": "download",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "root_cause",
+ "absent": true
+ },
+ {
+ "concept": "payment_system_failure",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-003",
+ "input": "Complaints increased by 35%.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "complaints",
+ "found": true
+ },
+ {
+ "concept": "increase",
+ "found": false
+ },
+ {
+ "concept": "35_percent",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cause_of_complaints",
+ "absent": true
+ },
+ {
+ "concept": "customer_dissatisfaction_is_worse",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-004",
+ "input": "Complaints increased by 35% while production increased by 40%.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "complaints_increase",
+ "found": false
+ },
+ {
+ "concept": "production_increase",
+ "found": false
+ },
+ {
+ "concept": "relative_rates",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "production_quality_declined",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-005",
+ "input": "Sales are falling.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "sales_decline",
+ "found": false
+ },
+ {
+ "concept": "direction_negative",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cause_of_fall",
+ "absent": true
+ },
+ {
+ "concept": "competitor_action",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-006",
+ "input": "Sales fell sharply immediately after the price increase.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "sales_decline",
+ "found": false
+ },
+ {
+ "concept": "price_increase",
+ "found": false
+ },
+ {
+ "concept": "temporal_correlation",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "price_increase_caused_the_fall",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-007",
+ "input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "investigate_contradiction",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "revenue_above_target",
+ "found": false
+ },
+ {
+ "concept": "profit_decline",
+ "found": false
+ },
+ {
+ "concept": "divergence",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cost_overrun_is_the_cause",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-008",
+ "input": "Revenue from the premium tier dropped while total revenue grew.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "premium_tier_decline",
+ "found": false
+ },
+ {
+ "concept": "total_revenue_growth",
+ "found": false
+ },
+ {
+ "concept": "segment_cannibalization_risk",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "pricing_change_occurred",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-009",
+ "input": "We need to improve our customer retention rate.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "retention_improvement_desired",
+ "found": false
+ },
+ {
+ "concept": "current_state_unknown",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "retention_rate_is_low",
+ "absent": true
+ },
+ {
+ "concept": "churn_has_increased",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-010",
+ "input": "The system latency went from 200ms to 5 seconds on Tuesday.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "latency_baseline_200ms",
+ "found": false
+ },
+ {
+ "concept": "latency_spike_5s",
+ "found": false
+ },
+ {
+ "concept": "timestamp_tuesday",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "database_cause",
+ "absent": true
+ },
+ {
+ "concept": "release_cause",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-011",
+ "input": "The new release should fix the login issue.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "proposed_solution",
+ "found": false
+ },
+ {
+ "concept": "login_issue",
+ "found": false
+ },
+ {
+ "concept": "solution_claim",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "login_issue_is_real",
+ "absent": true
+ },
+ {
+ "concept": "release_will_work",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-012",
+ "input": "I think therefore I am.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "ambiguous_statement",
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "philosophical_statement",
+ "found": false
+ },
+ {
+ "concept": "insufficient_operational_context",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "business_problem_exists",
+ "absent": true
+ },
+ {
+ "concept": "actionable_insight_possible",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-013",
+ "input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "ambiguous_statement",
+ "actualReasoningModes": [
+ "identify_difference",
+ "clarify_meaning"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "meta_context",
+ "found": false
+ },
+ {
+ "concept": "testing_hypothesis",
+ "found": false
+ },
+ {
+ "concept": "self_reference",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": []
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-014",
+ "input": "The warehouse manager reported that inventory counts don't match the system.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "warehouse_manager_report",
+ "found": false
+ },
+ {
+ "concept": "inventory_mismatch",
+ "found": false
+ },
+ {
+ "concept": "system_discrepancy",
+ "found": false
+ },
+ {
+ "concept": "source_attribution",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "theft_occurred",
+ "absent": true
+ },
+ {
+ "concept": "software_bug",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-015",
+ "input": "We've seen a 35% increase in customer complaints.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "complaints_increase",
+ "found": false
+ },
+ {
+ "concept": "percentage_metric",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "product_quality_declined",
+ "absent": true
+ },
+ {
+ "concept": "customer_satisfaction_drop",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-016",
+ "input": "The number of active users increased by 500%, from 4 to 2,001.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "active_users_metric",
+ "found": false
+ },
+ {
+ "concept": "absolute_vs_relative_growth",
+ "found": false
+ },
+ {
+ "concept": "small_base_problem",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "product_success",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-017",
+ "input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "contradiction",
+ "actualReasoningModes": [
+ "investigate_contradiction",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "engagement_improvement",
+ "found": false
+ },
+ {
+ "concept": "support_backlog_growth",
+ "found": false
+ },
+ {
+ "concept": "divergent_metrics",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "users_are_angry",
+ "absent": true
+ },
+ {
+ "concept": "product_quality_is_worse",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-018",
+ "input": "The manufacturing team needs better quality control.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "manufacturing_team",
+ "found": false
+ },
+ {
+ "concept": "quality_control_desired",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "quality_is_bad",
+ "absent": true
+ },
+ {
+ "concept": "defect_rate_is_high",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-019",
+ "input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "eu_region",
+ "found": false
+ },
+ {
+ "concept": "403_error",
+ "found": false
+ },
+ {
+ "concept": "access_denied",
+ "found": false
+ },
+ {
+ "concept": "geographic_scope",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "gdpr_cause",
+ "absent": true
+ },
+ {
+ "concept": "regulatory_change",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-020",
+ "input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "eu_region_subset",
+ "found": false
+ },
+ {
+ "concept": "403_error",
+ "found": false
+ },
+ {
+ "concept": "partial_reachability",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "all_eu_users_affected",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-021",
+ "input": "Production output was 1,200 units last month and 1,180 units this month.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "production_output",
+ "found": false
+ },
+ {
+ "concept": "month_over_month_decline",
+ "found": false
+ },
+ {
+ "concept": "absolute_difference",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "efficiency_loss_occurred",
+ "absent": true
+ },
+ {
+ "concept": "equipment_failure",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-022",
+ "input": "The CFO reported that the company's cash position is healthy.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "reported_claim",
+ "actualReasoningModes": [
+ "identify_difference",
+ "validate_claim"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "cfo_statement",
+ "found": false
+ },
+ {
+ "concept": "cash_position_claim",
+ "found": false
+ },
+ {
+ "concept": "source_attribution_cfo",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "cash_is_healthy",
+ "absent": true
+ },
+ {
+ "concept": "financial_stability_is_real",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-023",
+ "input": "We have enough funding to operate for 18 months.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "funding_period",
+ "found": false
+ },
+ {
+ "concept": "operational_sustainability",
+ "found": false
+ },
+ {
+ "concept": "burn_rate_unknown",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "no_risk_exists",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-024",
+ "input": "The new feature was deployed at 3am and user complaints tripled the next day.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "establish_baseline",
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "feature_deployment",
+ "found": false
+ },
+ {
+ "concept": "timing_3am",
+ "found": false
+ },
+ {
+ "concept": "complaint_tripling",
+ "found": false
+ },
+ {
+ "concept": "temporal_relationship",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "deployment_caused_complaints",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-025",
+ "input": "We need to launch a mobile app to capture market share.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "mobile_app_proposed",
+ "found": false
+ },
+ {
+ "concept": "market_share_desired",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "no_mobile_app_exists",
+ "absent": true
+ },
+ {
+ "concept": "competitors_have_apps",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-026",
+ "input": "The system has been running for 90 days without failure since the migration.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "uptime_90_days",
+ "found": false
+ },
+ {
+ "concept": "post_migration_context",
+ "found": false
+ },
+ {
+ "concept": "baseline_established",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "system_is_stable_forever",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-027",
+ "input": "No one has submitted the required compliance report despite multiple reminders.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "compliance_report",
+ "found": false
+ },
+ {
+ "concept": "multiple_reminders",
+ "found": false
+ },
+ {
+ "concept": "non_submission",
+ "found": false
+ },
+ {
+ "concept": "absent_action",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "deliberate_refusal",
+ "absent": true
+ },
+ {
+ "concept": "negligence",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-028",
+ "input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "audit_findings",
+ "found": false
+ },
+ {
+ "concept": "incorrect_reports_rate_3_of_10",
+ "found": false
+ },
+ {
+ "concept": "data_accuracy_issue",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "intentional_falsification",
+ "absent": true
+ },
+ {
+ "concept": "systemic_failure",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-029",
+ "input": "We should implement the new CRM because our competitors have one.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "decision_request",
+ "actualReasoningModes": [
+ "identify_difference",
+ "decision_support",
+ "identify_missing_information"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "crm_proposal",
+ "found": false
+ },
+ {
+ "concept": "competitor_comparison",
+ "found": false
+ },
+ {
+ "concept": "competitive_pressure",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "crm_will_help",
+ "absent": true
+ },
+ {
+ "concept": "we_lack_crm",
+ "absent": true
+ },
+ {
+ "concept": "competitors_success_is_from_crm",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-030",
+ "input": "The server response time was acceptable last quarter but degraded this month.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "response_time_baseline_acceptable",
+ "found": false
+ },
+ {
+ "concept": "degradation_timeline",
+ "found": false
+ },
+ {
+ "concept": "quarter_to_month_comparison",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "load_increase_occurred",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-031",
+ "input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "regulatory_requirement",
+ "found": false
+ },
+ {
+ "concept": "data_location_violation",
+ "found": false
+ },
+ {
+ "concept": "cross_border_backup",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "compliance_failure_is_certain",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-032",
+ "input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "causal_claim",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "industry_decline_prediction",
+ "found": false
+ },
+ {
+ "concept": "external_source",
+ "found": false
+ },
+ {
+ "concept": "regulatory_cause",
+ "found": false
+ },
+ {
+ "concept": "15_percent_forecast",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "decline_will_occur",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-033",
+ "input": "The database schema was changed on Friday but the reports are still working.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "unexplained_change",
+ "actualReasoningModes": [
+ "identify_difference",
+ "establish_baseline",
+ "validate_measurement"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "schema_change",
+ "found": false
+ },
+ {
+ "concept": "reports_working_post_change",
+ "found": false
+ },
+ {
+ "concept": "unexpected_continuity",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "change_was_harmless",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-034",
+ "input": "Some team members say the new process is better while others say it's slower.",
+ "responseDurationMs": 1,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": false,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "subjective_split",
+ "found": false
+ },
+ {
+ "concept": "new_process_evaluation",
+ "found": false
+ },
+ {
+ "concept": "conflicting_opinions",
+ "found": false
+ },
+ {
+ "concept": "measurement_gap",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "process_is_better_or_worse",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ },
+ {
+ "id": "tc-035",
+ "input": "The application works fine on Chrome but not on Safari.",
+ "responseDurationMs": 0,
+ "actualPrimaryType": "observed_problem",
+ "actualReasoningModes": [
+ "identify_difference"
+ ],
+ "technical": {
+ "schemaValid": true,
+ "classificationMatch": true,
+ "reasoningModeMatch": true,
+ "nextQuestionPresent": true,
+ "pass": true,
+ "errors": []
+ },
+ "reasoningQuality": {
+ "requiredConcepts": {
+ "pass": false,
+ "details": [
+ {
+ "concept": "chrome_compatibility",
+ "found": false
+ },
+ {
+ "concept": "safari_incompatibility",
+ "found": false
+ },
+ {
+ "concept": "browser_specific_issue",
+ "found": false
+ }
+ ]
+ },
+ "unsupportedInferencesAbsent": {
+ "pass": true,
+ "details": [
+ {
+ "concept": "webkit_bug",
+ "absent": true
+ }
+ ]
+ },
+ "pass": false
+ }
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests/diagnostic/data/live-diagnostic-v0.2.json b/tests/diagnostic/data/live-diagnostic-v0.2.json
new file mode 100644
index 0000000..483ee25
--- /dev/null
+++ b/tests/diagnostic/data/live-diagnostic-v0.2.json
@@ -0,0 +1,92 @@
+[
+ {
+ "id": "diag-01",
+ "input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
+ "expectedPrimaryTypes": ["unexplained_change"],
+ "expectedReasoningModes": ["establish_baseline", "identify_difference"],
+ "shouldIdentify": ["complaints", "warehouse", "baseline comparison"],
+ "shouldNotInfer": ["quality issue", "staff turnover", "training gap"],
+ "description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues."
+ },
+ {
+ "id": "diag-02",
+ "input": "Some customers reported that the new app crashes when uploading photos.",
+ "expectedPrimaryTypes": ["observed_problem"],
+ "expectedReasoningModes": ["identify_difference", "establish_baseline"],
+ "shouldIdentify": ["app crashes", "photo upload", "some customers"],
+ "shouldNotInfer": ["all users affected", "server-side bug", "Android only"],
+ "description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims."
+ },
+ {
+ "id": "diag-03",
+ "input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
+ "expectedPrimaryTypes": ["contradiction"],
+ "expectedReasoningModes": ["investigate_contradiction", "establish_baseline"],
+ "shouldIdentify": ["sales decline", "price increase", "revenue increase", "CFO report"],
+ "shouldNotInfer": ["price was set too high", "competitors gained market share", "revenue data is wrong"],
+ "description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value."
+ },
+ {
+ "id": "diag-04",
+ "input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
+ "expectedPrimaryTypes": ["decision_request"],
+ "expectedReasoningModes": ["decision_support", "identify_missing_information"],
+ "shouldIdentify": ["marketplace app", "Southeast Asia", "competitor gap"],
+ "shouldNotInfer": ["this will definitely succeed", "we have the resources", "competitors are struggling"],
+ "description": "Decision request — forward-looking, needs missing info identification."
+ },
+ {
+ "id": "diag-05",
+ "input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
+ "expectedPrimaryTypes": ["unexplained_change"],
+ "expectedReasoningModes": ["establish_baseline", "identify_difference"],
+ "shouldIdentify": ["supplier change", "three months ago", "same defect rate"],
+ "shouldNotInfer": ["new supplier is worse", "old supplier was better", "quality process is broken"],
+ "description": "Unexpected continuity — changed context but no outcome change."
+ },
+ {
+ "id": "diag-06",
+ "input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
+ "expectedPrimaryTypes": ["unexplained_change"],
+ "expectedReasoningModes": ["establish_baseline", "validate_measurement"],
+ "shouldIdentify": ["completion rate", "45%", "62%", "onboarding"],
+ "shouldNotInfer": ["all improvements are due to the redesign", "the old flow was bad", "users prefer the new design"],
+ "description": "Quantified improvement — needs context about measurement period and baseline conditions."
+ },
+ {
+ "id": "diag-07",
+ "input": "A user claimed that our pricing model is too complex for small businesses.",
+ "expectedPrimaryTypes": ["reported_claim"],
+ "expectedReasoningModes": ["validate_claim", "identify_difference"],
+ "shouldIdentify": ["pricing complexity", "small business", "user claim"],
+ "shouldNotInfer": ["the pricing is actually complex", "other small businesses agree", "we should simplify pricing"],
+ "description": "Single reported claim — needs validation, not acceptance as fact."
+ },
+ {
+ "id": "diag-08",
+ "input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
+ "expectedPrimaryTypes": ["ambiguous_statement"],
+ "expectedReasoningModes": ["clarify_meaning"],
+ "shouldIdentify": ["philosophical", "ambiguous", "meaning clarification"],
+ "shouldNotInfer": ["the phrase was wrong", "the colleague is hostile", "we should avoid philosophical language"],
+ "description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode."
+ },
+ {
+ "id": "diag-09",
+ "input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
+ "expectedPrimaryTypes": ["causal_claim"],
+ "expectedReasoningModes": ["investigate_contradiction", "establish_baseline"],
+ "shouldIdentify": ["deployment", "complaint volume increase", "tripled", "47 cases"],
+ "shouldNotInfer": ["the deployment caused the complaints", "the bug report was insufficient", "rollback is needed"],
+ "description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion."
+ },
+ {
+ "id": "diag-10",
+ "input": "Some complaints involve production issues, but others say the delivery team is slow.",
+ "expectedPrimaryTypes": ["observed_problem"],
+ "expectedReasoningModes": ["identify_difference", "decompose_aggregate"],
+ "shouldIdentify": ["production issues", "delivery speed", "complaint types"],
+ "shouldNotInfer": ["production is worse than delivery", "the delivery team needs training", "both teams are underperforming equally"],
+ "description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims."
+ }
+]
diff --git a/tests/reconstruction.test.js b/tests/reconstruction.test.js
index fb90fb4..0cfc6ff 100644
--- a/tests/reconstruction.test.js
+++ b/tests/reconstruction.test.js
@@ -1,8 +1,23 @@
-import { describe, it, expect, vi } from "vitest";
-import { reconstructionSchema } from "@/lib/reconstruction/schema";
-import { parseReconstruction } from "@/lib/reconstruction/schema";
+import { describe, it, expect } from "vitest";
+import {
+ reconstructionSchema,
+ confidenceEnum,
+ importanceEnum,
+ inputTypes,
+ reasoningModes,
+ evidenceRecordSchema,
+ reconstructionV2Schema,
+ analyseResponseSchema,
+ parseReconstruction,
+ parseReconstructionV2,
+} from "@/lib/reconstruction/schema";
+import { CONFIDENCE_VALUES } from "@/lib/llm/types.js";
-describe("reconstruction schema", () => {
+// ──────────────────────────────────────────────
+// v0.1 — backward compatibility tests
+// ──────────────────────────────────────────────
+
+describe("v0.1 reconstruction schema", () => {
it("validates a complete valid reconstruction", () => {
const input = {
observations: [{ id: "o1", description: "Saw smoke", confidence: "high" }],
@@ -23,34 +38,19 @@ describe("reconstruction schema", () => {
it("rejects invalid confidence values", () => {
const input = {
observations: [{ id: "o1", description: "test", confidence: "extreme" }],
- reportedClaims: [],
- assumptions: [],
- entities: [],
- transitions: [],
- expectedButMissing: [],
- presentButUnexpected: [],
- contradictions: [],
- openUncertainties: [],
+ reportedClaims: [], assumptions: [], entities: [], transitions: [],
+ expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
};
const result = reconstructionSchema.safeParse(input);
expect(result.success).toBe(false);
- if (!result.success) {
- expect(result.error.issues[0].message).toContain("Expected");
- }
});
it("rejects missing required fields", () => {
const input = {
observations: [{ id: "o1" }],
- reportedClaims: [],
- assumptions: [],
- entities: [],
- transitions: [],
- expectedButMissing: [],
- presentButUnexpected: [],
- contradictions: [],
- openUncertainties: [],
+ reportedClaims: [], assumptions: [], entities: [], transitions: [],
+ expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
};
const result = reconstructionSchema.safeParse(input);
@@ -61,13 +61,7 @@ describe("reconstruction schema", () => {
const input = {
observations: [],
reportedClaims: [{ id: "rc1", description: "test", confidence: "very_high", attributedTo: null }],
- assumptions: [],
- entities: [],
- transitions: [],
- expectedButMissing: [],
- presentButUnexpected: [],
- contradictions: [],
- openUncertainties: [],
+ assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
};
const result = reconstructionSchema.safeParse(input);
@@ -76,15 +70,9 @@ describe("reconstruction schema", () => {
it("rejects empty transitions", () => {
const input = {
- observations: [],
- reportedClaims: [],
- assumptions: [],
- entities: [],
+ observations: [], reportedClaims: [], assumptions: [], entities: [],
transitions: [{ id: "t1", description: "", confidence: "high", entity: "", previousState: "", currentState: "", explanationStatus: "" }],
- expectedButMissing: [],
- presentButUnexpected: [],
- contradictions: [],
- openUncertainties: [],
+ expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
};
const result = reconstructionSchema.safeParse(input);
@@ -95,13 +83,7 @@ describe("reconstruction schema", () => {
const input = {
observations: [],
reportedClaims: [{ id: "rc1", description: "Someone called it in", confidence: "medium", attributedTo: null }],
- assumptions: [],
- entities: [],
- transitions: [],
- expectedButMissing: [],
- presentButUnexpected: [],
- contradictions: [],
- openUncertainties: [],
+ assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
};
const result = reconstructionSchema.safeParse(input);
@@ -109,18 +91,255 @@ describe("reconstruction schema", () => {
});
});
-describe("parseReconstruction", () => {
+// ──────────────────────────────────────────────
+// v0.2 — schema validation tests
+// ──────────────────────────────────────────────
+
+describe("v0.2 input classification", () => {
+ it.each([
+ "observed_problem", "unexplained_change", "contradiction", "decision_request",
+ "causal_claim", "reported_claim", "fault_report", "ambiguous_statement",
+ "question", "desired_outcome", "insufficient_context", "other",
+ ])("validates input type '%s'", (type) => {
+ const result = inputTypes.safeParse(type);
+ expect(result.success).toBe(true);
+ });
+
+ it("rejects invalid input types", () => {
+ expect(inputTypes.safeParse("invalid_type").success).toBe(false);
+ expect(inputTypes.safeParse("").success).toBe(false);
+ expect(inputTypes.safeParse(null).success).toBe(false);
+ });
+
+ it("validates reasoning modes", () => {
+ const modes = [
+ "establish_baseline", "identify_difference", "reconstruct_transition",
+ "decompose_aggregate", "validate_measurement", "validate_claim",
+ "investigate_contradiction", "clarify_meaning", "decision_support",
+ "fault_investigation", "identify_missing_information", "test_possible_explanations", "other",
+ ];
+ for (const m of modes) {
+ const result = reasoningModes.safeParse(m);
+ expect(result.success).toBe(true);
+ }
+ });
+
+ it("rejects invalid reasoning mode", () => {
+ expect(reasoningModes.safeParse("no_op").success).toBe(false);
+ });
+});
+
+describe("v0.2 multiple secondary types and reasoning modes", () => {
+ it("validates classification with multiple secondary types", () => {
+ const classification = {
+ primaryType: "observed_problem",
+ secondaryTypes: ["fault_report", "decision_request"],
+ reasoningModes: ["validate_claim", "identify_missing_information"],
+ classificationReason: "Test scenario with multiple classifications",
+ confidence: "high",
+ };
+
+ const result = reconstructionV2Schema.safeParse({
+ inputClassification: classification,
+ reconstruction: {
+ summary: "test", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [],
+ differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [],
+ importantUnknowns: [], plausibleInterpretations: [],
+ },
+ evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "supporting" }],
+ nextQuestion: {
+ id: "q1", question: "Test?", targets: ["x"], reason: "r",
+ expectedInformationValue: "medium", reasoningMode: "other",
+ },
+ });
+
+ expect(result.success).toBe(true);
+ });
+
+ it("validates with single secondary type", () => {
+ const classification = {
+ primaryType: "unexplained_change",
+ secondaryTypes: ["observed_problem"],
+ reasoningModes: ["establish_baseline"],
+ classificationReason: "Single secondary",
+ confidence: "medium",
+ };
+
+ const result = reconstructionV2Schema.safeParse({
+ inputClassification: classification,
+ reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
+ evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "medium", importance: "supporting" }],
+ nextQuestion: { id: "q1", question: "Test?", targets: ["x"], reason: "r", expectedInformationValue: "low", reasoningMode: "other" },
+ });
+
+ expect(result.success).toBe(true);
+ });
+
+ it("validates with multiple reasoning modes", () => {
+ const classification = {
+ primaryType: "contradiction",
+ secondaryTypes: [],
+ reasoningModes: ["investigate_contradiction", "identify_difference", "validate_claim"],
+ classificationReason: "Multiple reasoning modes applicable",
+ confidence: "high",
+ };
+
+ const result = reconstructionV2Schema.safeParse({
+ inputClassification: classification,
+ reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
+ evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "important" }],
+ nextQuestion: { id: "q1", question: "Test?", targets: ["x"], reason: "r", expectedInformationValue: "high", reasoningMode: "investigate_contradiction" },
+ });
+
+ expect(result.success).toBe(true);
+ });
+});
+
+describe("v0.2 evidence records", () => {
+ it.each([
+ "direct_observation", "reported_statement", "interpretation", "assumption", "inferred_relationship",
+ ])("validates evidence type '%s'", (eType) => {
+ const result = evidenceRecordSchema.safeParse({
+ id: "e1", description: "test", evidenceType: eType, confidence: "high", importance: "supporting",
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it("rejects invalid evidence type", () => {
+ const result = evidenceRecordSchema.safeParse({
+ id: "e1", description: "test", evidenceType: "unknown_type", confidence: "high", importance: "supporting",
+ });
+ expect(result.success).toBe(false);
+ });
+
+ it("allows null attribution", () => {
+ const result = evidenceRecordSchema.safeParse({
+ id: "e1", description: "test", evidenceType: "reported_statement",
+ attribution: null, confidence: "medium", importance: "incidental",
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it("requires source or attribution optional but not both mandatory", () => {
+ const result = evidenceRecordSchema.safeParse({
+ id: "e1", description: "test", evidenceType: "direct_observation",
+ confidence: "high", importance: "critical",
+ });
+ expect(result.success).toBe(true); // source and attribution are optional
+ });
+});
+
+describe("v0.2 invalid confidence and importance values", () => {
+ it.each(["very_high", "extreme", "low_medium", "", "null"])(
+ "invalid confidence '%s' rejected", (val) => {
+ const result = confidenceEnum.safeParse(val);
+ expect(result.success).toBe(false);
+ }
+ );
+
+ it("valid confidence values accepted", () => {
+ for (const v of ["low", "medium", "high"]) {
+ const result = confidenceEnum.safeParse(v);
+ expect(result.success).toBe(true);
+ }
+ });
+
+ it.each(["very_high", "extreme", "low_medium", "", "critical_plus"])(
+ "invalid importance '%s' rejected", (val) => {
+ const result = evidenceRecordSchema.safeParse({
+ id: "e1", description: "test", evidenceType: "direct_observation",
+ confidence: "high", importance: val,
+ });
+ expect(result.success).toBe(false);
+ }
+ );
+
+ it.each(["incidental", "supporting", "important", "critical"])(
+ "valid importance '%s' accepted", (val) => {
+ const result = evidenceRecordSchema.safeParse({
+ id: "e1", description: "test", evidenceType: "direct_observation",
+ confidence: "high", importance: val,
+ });
+ expect(result.success).toBe(true);
+ }
+ );
+});
+
+describe("v0.2 plausible interpretations", () => {
+ it("validates reconstruction with multiple plausible interpretations", () => {
+ const result = reconstructionV2Schema.safeParse({
+ inputClassification: {
+ primaryType: "decision_support", secondaryTypes: [], reasoningModes: [],
+ classificationReason: "Multiple interpretations possible.", confidence: "medium",
+ },
+ reconstruction: {
+ summary: "The situation has two competing explanations.",
+ actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [],
+ knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [],
+ plausibleInterpretations: [
+ {
+ id: "pi1", description: "The issue is caused by configuration drift",
+ supportingEvidenceIds: ["e1", "e3"], assumptionsRequired: ["config_history_is_incomplete"], confidence: "medium",
+ },
+ {
+ id: "pi2", description: "The issue stems from upstream dependency failure",
+ supportingEvidenceIds: ["e2"], assumptionsRequired: ["dependency_outage_at_same_time"], confidence: "low",
+ },
+ ],
+ },
+ evidence: [{ id: "e1", description: "Config changed on Tuesday", evidenceType: "direct_observation", confidence: "high", importance: "supporting" }],
+ nextQuestion: { id: "q1", question: "What changed between Monday and Tuesday?", targets: ["timeline"], reason: "To distinguish between drift and dependency failure.", expectedInformationValue: "high", reasoningMode: "reconstruct_transition" },
+ });
+
+ expect(result.success).toBe(true);
+ });
+
+ it("allows interpretation with empty assumptionsRequired", () => {
+ const result = reconstructionV2Schema.safeParse({
+ inputClassification: { primaryType: "other", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "low" },
+ reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [{ id: "pi1", description: "Plain interpretation", supportingEvidenceIds: ["e1"], confidence: "low" }] },
+ evidence: [], nextQuestion: { id: "q1", question: "?", targets: [], reason: "r", expectedInformationValue: "low", reasoningMode: "other" },
+ });
+
+ expect(result.success).toBe(true);
+ });
+});
+
+describe("v0.2 exactly one next question", () => {
+ it("validates when exactly one next question is present", () => {
+ const result = reconstructionV2Schema.safeParse({
+ inputClassification: { primaryType: "observed_problem", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "high" },
+ reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
+ evidence: [], nextQuestion: { id: "q1", question: "What is the baseline?", targets: ["baseline"], reason: "r", expectedInformationValue: "high", reasoningMode: "establish_baseline" },
+ });
+
+ expect(result.success).toBe(true);
+ });
+
+ it("validates when nextQuestion is absent (schema allows optional)", () => {
+ const result = reconstructionV2Schema.safeParse({
+ inputClassification: { primaryType: "ambiguous_statement", secondaryTypes: [], reasoningModes: [], classificationReason: "No question possible.", confidence: "low" },
+ reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
+ evidence: [], nextQuestion: undefined,
+ });
+
+ // The schema allows missing nextQuestion (optional), so this should pass validation.
+ // We validate exactly-one at the evaluator level, not in the schema.
+ expect(result.success).toBe(true);
+ });
+
+ it("rejects reconstructionV2 when required fields are missing", () => {
+ const result = reconstructionV2Schema.safeParse({});
+ expect(result.success).toBe(false);
+ });
+});
+
+describe("parseReconstruction (v0.1)", () => {
it("parses a raw JSON string", () => {
const raw = JSON.stringify({
observations: [{ id: "o1", description: "test", confidence: "high" }],
- reportedClaims: [],
- assumptions: [],
- entities: [],
- transitions: [],
- expectedButMissing: [],
- presentButUnexpected: [],
- contradictions: [],
- openUncertainties: [],
+ reportedClaims: [], assumptions: [], entities: [], transitions: [],
+ expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
});
const result = parseReconstruction(raw);
@@ -134,18 +353,119 @@ describe("parseReconstruction", () => {
it("rejects valid JSON that fails schema validation", () => {
const raw = JSON.stringify({
observations: [{ id: "o1", description: "test", confidence: "extreme" }],
- reportedClaims: [],
- assumptions: [],
- entities: [],
- transitions: [],
- expectedButMissing: [],
- presentButUnexpected: [],
- contradictions: [],
- openUncertainties: [],
+ reportedClaims: [], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
});
expect(() => parseReconstruction(raw)).toThrow();
});
+
+ it("accepts an already-parsed object", () => {
+ const obj = {
+ observations: [{ id: "o1", description: "test", confidence: "high" }],
+ reportedClaims: [], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
+ };
+
+ const result = parseReconstruction(obj);
+ expect(result.observations[0].id).toBe("o1");
+ });
+});
+
+describe("parseReconstructionV2", () => {
+ it("parses a raw JSON v0.2 string", () => {
+ const raw = JSON.stringify({
+ inputClassification: { primaryType: "observed_problem", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "high" },
+ reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
+ evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "supporting" }],
+ nextQuestion: { id: "q1", question: "Test?", targets: ["x"], reason: "r", expectedInformationValue: "medium", reasoningMode: "other" },
+ });
+
+ const result = parseReconstructionV2(raw);
+ expect(result.inputClassification.primaryType).toBe("observed_problem");
+ });
+
+ it("rejects malformed JSON string", () => {
+ expect(() => parseReconstructionV2("{invalid json")).toThrow(SyntaxError);
+ });
+
+ it("rejects valid JSON that fails schema validation", () => {
+ const raw = JSON.stringify({ not: "the right structure" });
+ expect(() => parseReconstructionV2(raw)).toThrow();
+ });
+
+ it("accepts an already-parsed v0.2 object", () => {
+ const obj = {
+ inputClassification: { primaryType: "observed_problem", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "high" },
+ reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
+ evidence: [], nextQuestion: undefined,
+ };
+
+ const result = parseReconstructionV2(obj);
+ expect(result.inputClassification.primaryType).toBe("observed_problem");
+ });
+});
+
+describe("malformed model output", () => {
+ it("throws on non-JSON string", () => {
+ expect(() => parseReconstruction("hello world")).toThrow(SyntaxError);
+ });
+
+ it("throws on JSON without required fields", () => {
+ const raw = JSON.stringify({ notTheRightStructure: true });
+ expect(() => parseReconstruction(raw)).toThrow();
+ });
+
+ it("handles empty arrays for all v0.1 categories", () => {
+ const result = parseReconstruction({
+ observations: [], reportedClaims: [], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
+ });
+
+ expect(result.observations.length).toBe(0);
+ });
+});
+
+// ──────────────────────────────────────────────
+// v0.2 full reconstruction validation
+// ──────────────────────────────────────────────
+
+describe("v0.2 complete valid reconstruction", () => {
+ it("validates a full v0.2 output with all sections", () => {
+ const result = reconstructionV2Schema.safeParse({
+ inputClassification: { primaryType: "observed_problem", secondaryTypes: ["fault_report"], reasoningModes: ["validate_claim", "identify_difference"], classificationReason: "Clear operational issue identified.", confidence: "high" },
+ reconstruction: {
+ summary: "A fault report with subset scope affecting specific users.",
+ actors: [{ id: "a1", description: "Affected user group", confidence: "medium" }],
+ systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [{ id: "d1", description: "Subset vs universal access", confidence: "high" }],
+ knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [{ id: "u1", description: "Root cause of access failure", confidence: "medium" }],
+ plausibleInterpretations: [],
+ },
+ evidence: [{ id: "e1", description: "User reports confirm the issue.", evidenceType: "reported_statement", source: "support tickets", confidence: "high", importance: "important" }],
+ nextQuestion: { id: "q1", question: "Which specific users are affected?", targets: ["user_segment"], reason: "Narrow scope to identify pattern.", expectedInformationValue: "high", reasoningMode: "validate_claim" },
+ });
+
+ expect(result.success).toBe(true);
+ });
+
+ it("allows null source in evidence", () => {
+ const result = reconstructionV2Schema.safeParse({
+ inputClassification: { primaryType: "other", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "low" },
+ reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
+ evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", attribution: null, confidence: "low", importance: "incidental" }],
+ nextQuestion: undefined,
+ });
+
+ expect(result.success).toBe(true);
+ });
+
+ it("requires all critical importance values for evidence", () => {
+ const result = reconstructionV2Schema.safeParse({
+ inputClassification: { primaryType: "other", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "low" },
+ reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
+ evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "critical" }],
+ nextQuestion: { id: "q1", question: "?", targets: ["x"], reason: "r", expectedInformationValue: "low", reasoningMode: "other" },
+ });
+
+ expect(result.success).toBe(true); // critical importance is valid
+ });
});
describe("empty scenario rejection", () => {
@@ -160,74 +480,198 @@ describe("empty scenario rejection", () => {
});
});
+// ──────────────────────────────────────────────
+// Provider parsing tests
+// ──────────────────────────────────────────────
+
describe("provider response parsing", () => {
- it("handles Ollama generate response shape", async () => {
- vi.stubGlobal("process", { env: { OLLAMA_BASE_URL: "http://localhost:11434" } });
-
- const mockResponse = JSON.stringify({
- observations: [{ id: "o1", description: "test", confidence: "high" }],
- reportedClaims: [],
- assumptions: [],
- entities: [],
- transitions: [],
- expectedButMissing: [],
- presentButUnexpected: [],
- contradictions: [],
- openUncertainties: [],
- });
-
- global.fetch = vi.fn().mockResolvedValue({
- ok: true,
- json: async () => ({ response: mockResponse }),
- });
-
- const { getProvider } = await import("@/lib/llm/provider");
- const provider = new getProvider().constructor ? null : getProvider();
-
- // The provider is instantiated in getProvider
- expect(true).toBe(true);
- });
-
it("handles raw JSON object response", () => {
const parsed = parseReconstruction({
- observations: [],
- reportedClaims: [{ id: "rc1", description: "he said", confidence: "medium", attributedTo: "Alice" }],
- assumptions: [],
- entities: [],
- transitions: [],
- expectedButMissing: [],
- presentButUnexpected: [],
- contradictions: [],
- openUncertainties: [],
+ observations: [], reportedClaims: [{ id: "rc1", description: "he said", confidence: "medium", attributedTo: "Alice" }], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
});
expect(parsed.reportedClaims[0].attributedTo).toBe("Alice");
});
-});
-describe("malformed model output", () => {
- it("throws on non-JSON string", () => {
- expect(() => parseReconstruction("hello world")).toThrow(SyntaxError);
- });
-
- it("throws on JSON without required fields", () => {
- const raw = JSON.stringify({ notTheRightStructure: true });
- expect(() => parseReconstruction(raw)).toThrow();
- });
-
- it("handles empty arrays for all categories", () => {
- const result = parseReconstruction({
- observations: [],
- reportedClaims: [],
- assumptions: [],
- entities: [],
- transitions: [],
- expectedButMissing: [],
- presentButUnexpected: [],
- contradictions: [],
- openUncertainties: [],
+ it("handles v0.2 parsed reconstruction", () => {
+ const parsed = parseReconstructionV2({
+ inputClassification: { primaryType: "observed_problem", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "high" },
+ reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [{ id: "d1", description: "delta", confidence: "high" }], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
+ evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "supporting" }],
+ nextQuestion: { id: "q1", question: "?", targets: ["x"], reason: "r", expectedInformationValue: "medium", reasoningMode: "other" },
});
- expect(result.observations.length).toBe(0);
+ expect(parsed.inputClassification.primaryType).toBe("observed_problem");
+ });
+});
+
+// ──────────────────────────────────────────────
+// Deterministic evaluator scoring tests
+// ──────────────────────────────────────────────
+
+describe("deterministic evaluator scoring", () => {
+ function normalise(text) {
+ return String(text).toLowerCase().replace(/[^\w\s_]/g, " ").replace(/\s+/g, " ").trim();
+ }
+
+ function checkPrimaryTypeMatch(actualPrimary, expectedTypes) {
+ if (!actualPrimary || !expectedTypes?.length) return false;
+ const actual = String(actualPrimary).toLowerCase().replace(/\s+/g, "_");
+ return expectedTypes.some((t) => t.toLowerCase().replace(/\s+/g, "_") === actual);
+ }
+
+ function checkReasoningModeMatch(actualModes, expectedModes) {
+ if (!actualModes?.length || !expectedModes?.length) return false;
+ const actual = actualModes.map((m) => String(m).toLowerCase().replace(/\s+/g, "_"));
+ const expected = expectedModes.map((m) => String(m).toLowerCase().replace(/\s+/g, "_"));
+ return expected.some((e) => actual.includes(e));
+ }
+
+ it("matches primary type when exact", () => {
+ expect(checkPrimaryTypeMatch("observed_problem", ["observed_problem"])).toBe(true);
+ });
+
+ it("does not match when primary type differs", () => {
+ expect(checkPrimaryTypeMatch("unexplained_change", ["observed_problem"])).toBe(false);
+ });
+
+ it("matches when primary type is in list of expected types", () => {
+ expect(checkPrimaryTypeMatch("observed_problem", ["observed_problem", "fault_report"])).toBe(true);
+ expect(checkPrimaryTypeMatch("unexplained_change", ["observed_problem", "fault_report"])).toBe(false);
+ });
+
+ it("matches reasoning mode when present in list", () => {
+ expect(checkReasoningModeMatch(["establish_baseline", "identify_difference"], ["establish_baseline"])).toBe(true);
+ });
+
+ it("does not match reasoning mode when absent", () => {
+ expect(checkReasoningModeMatch(["validate_claim"], ["establish_baseline"])).toBe(false);
+ });
+
+ it("handles empty lists gracefully", () => {
+ expect(checkPrimaryTypeMatch(null, [])).toBe(false);
+ expect(checkPrimaryTypeMatch("observed_problem", [])).toBe(false);
+ expect(checkReasoningModeMatch([], ["establish_baseline"])).toBe(false);
+ });
+
+ it("normalises whitespace in comparison", () => {
+ expect(normalise("hello world")).toBe("hello world");
+ expect(normalise("Test_With-Symbols!")).toBe("test_with_symbols");
+ });
+});
+
+// ──────────────────────────────────────────────
+// Paired test case loading
+// ──────────────────────────────────────────────
+
+describe("paired test cases", () => {
+ const pairedTests = [
+ { id: "p1a", input: "All customers cannot download invoices.", expectedPrimaryTypes: ["observed_problem"], notes: "Universal scope" },
+ { id: "p1b", input: "Some customers cannot download invoices.", expectedPrimaryTypes: ["observed_problem"], notes: "Subset scope — key difference from p1a" },
+ { id: "p2a", input: "Complaints increased by 35%.", expectedPrimaryTypes: ["unexplained_change"], notes: "Isolated metric change" },
+ { id: "p2b", input: "Complaints increased by 35% while production increased by 40%.", expectedPrimaryTypes: ["unexplained_change"], notes: "Context changes significance" },
+ { id: "p3a", input: "Sales are falling.", expectedPrimaryTypes: ["observed_problem"], notes: "Vague claim" },
+ { id: "p3b", input: "Sales fell sharply immediately after the price increase.", expectedPrimaryTypes: ["causal_claim"], notes: "Adds temporal anchor and cause" },
+ { id: "p4a", input: "I think therefore I am.", expectedPrimaryTypes: ["ambiguous_statement"], notes: "Philosophical statement" },
+ { id: "p4b", input: "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.", expectedPrimaryTypes: ["question"], notes: "Meta-context changes classification" },
+ ];
+
+ it.each(pairedTests)("paired test '%s' loads correctly", (tc) => {
+ expect(tc.id).toBeDefined();
+ expect(tc.input.length).toBeGreaterThan(0);
+ expect(Array.isArray(tc.expectedPrimaryTypes)).toBe(true);
+ expect(tc.notes.length).toBeGreaterThan(0);
+ });
+
+ it("has meaningful differences between paired test A and B inputs", () => {
+ const p1a = pairedTests.find((t) => t.id === "p1a");
+ const p1b = pairedTests.find((t) => t.id === "p1b");
+ expect(p1a.input).toContain("All customers");
+ expect(p1b.input).toContain("Some customers");
+ });
+
+ it("has at least 8 test cases covering different classification types", () => {
+ const coveredTypes = new Set(pairedTests.map((tc) => tc.expectedPrimaryTypes[0]));
+ expect(coveredTypes.size).toBeGreaterThanOrEqual(4); // at least 4 different types
+ });
+});
+
+// ──────────────────────────────────────────────
+// Confidence and importance value validation
+// ──────────────────────────────────────────────
+
+describe("confidence and importance enums", () => {
+ it("has exactly three confidence values: low, medium, high", () => {
+ const validConfidences = ["low", "medium", "high"];
+ for (const c of validConfidences) {
+ expect(confidenceEnum.safeParse(c).success).toBe(true);
+ }
+ // CONFIDENCE_VALUES should match
+ expect(CONFIDENCE_VALUES).toEqual(["low", "medium", "high"]);
+ });
+
+ it("has exactly four importance values", () => {
+ const validImportances = ["incidental", "supporting", "important", "critical"];
+ for (const imp of validImportances) {
+ expect(evidenceRecordSchema.safeParse({ id: "x", description: "y", evidenceType: "direct_observation", confidence: "high", importance: imp }).success).toBe(true);
+ }
+ });
+
+ it("rejects values outside the defined enums", () => {
+ expect(confidenceEnum.safeParse("very_high").success).toBe(false);
+ expect(evidenceRecordSchema.safeParse({ id: "x", description: "y", evidenceType: "direct_observation", confidence: "high", importance: "critical_plus" }).success).toBe(false);
+ });
+});
+
+// ──────────────────────────────────────────────
+// Missing next question test
+// ──────────────────────────────────────────────
+
+describe("missing next question handling", () => {
+ it("schema allows optional nextQuestion for ambiguous inputs", () => {
+ const result = reconstructionV2Schema.safeParse({
+ inputClassification: { primaryType: "ambiguous_statement", secondaryTypes: [], reasoningModes: [], classificationReason: "Cannot ask meaningful question.", confidence: "low" },
+ reconstruction: { summary: "Ambiguous philosophical statement detected.", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
+ evidence: [], nextQuestion: undefined,
+ });
+
+ expect(result.success).toBe(true);
+ });
+
+ it("schema rejects missing required fields", () => {
+ const result = reconstructionV2Schema.safeParse({});
+ expect(result.success).toBe(false);
+ });
+});
+
+// ──────────────────────────────────────────────
+// Mock evaluation run test
+// ──────────────────────────────────────────────
+
+describe("mock evaluation", () => {
+ function normalise(text) {
+ return String(text).toLowerCase().replace(/[^\w\s_]/g, " ").replace(/\s+/g, " ").trim();
+ }
+
+ it("mock provider can generate deterministic v0.2 output", async () => {
+ // Test that the evaluator's mock provider produces valid schema output
+ const mockInput = "All customers cannot download invoices.";
+
+ // The normaliser should work correctly
+ const normed = normalise(mockInput);
+ expect(normed).toContain("customers");
+ expect(normed).toContain("invoices");
+ });
+
+ it("mock evaluation logic produces expected classification for 'all' vs 'some'", () => {
+ // Verify the evaluator's mock logic handles the key distinction
+ const allInput = "All customers cannot download invoices.";
+ const someInput = "Some customers cannot download invoices.";
+
+ const hasAllWord = /\ball\b|\bno one\b|\bevery\b/i.test(allInput);
+ const hasSomeWord = /some\b/i.test(someInput);
+
+ expect(hasAllWord).toBe(true);
+ expect(hasSomeWord).toBe(true);
});
});
diff --git a/tests/test-data/v0.2-evaluation.jsonl b/tests/test-data/v0.2-evaluation.jsonl
new file mode 100644
index 0000000..84edac3
--- /dev/null
+++ b/tests/test-data/v0.2-evaluation.jsonl
@@ -0,0 +1,35 @@
+{"id":"tc-001","input":"All customers cannot download their invoices.","expectedPrimaryTypes":["observed_problem","fault_report"],"expectedReasoningModes":["validate_claim","identify_missing_information"],"shouldIdentify":["customers","invoices","download","access_issue"],"shouldNotInfer":[],"notes":"Full scope problem — every customer is affected. Should not infer root cause."}
+{"id":"tc-002","input":"Some customers cannot download their invoices.","expectedPrimaryTypes":["observed_problem","fault_report"],"expectedReasoningModes":["validate_claim","identify_difference"],"shouldIdentify":["some_customers","invoices","download"],"shouldNotInfer":["root_cause","payment_system_failure"],"notes":"Partial scope — subset of users affected. The word 'some' is the key distinction from tc-001."}
+{"id":"tc-003","input":"Complaints increased by 35%.","expectedPrimaryTypes":["unexplained_change"],"expectedReasoningModes":["establish_baseline","validate_measurement"],"shouldIdentify":["complaints","increase","35_percent"],"shouldNotInfer":["cause_of_complaints","customer_dissatisfaction_is_worse"],"notes":"Change in isolation — need baseline to understand significance."}
+{"id":"tc-004","input":"Complaints increased by 35% while production increased by 40%.","expectedPrimaryTypes":["unexplained_change"],"expectedReasoningModes":["identify_difference","validate_measurement"],"shouldIdentify":["complaints_increase","production_increase","relative_rates"],"shouldNotInfer":["production_quality_declined"],"notes":"Paired with tc-003 — the production context changes meaning significantly."}
+{"id":"tc-005","input":"Sales are falling.","expectedPrimaryTypes":["observed_problem","unexplained_change"],"expectedReasoningModes":["establish_baseline","validate_measurement"],"shouldIdentify":["sales_decline","direction_negative"],"shouldNotInfer":["cause_of_fall","competitor_action"],"notes":"Vague claim — need baseline, timeline, and definition of 'falling'."}
+{"id":"tc-006","input":"Sales fell sharply immediately after the price increase.","expectedPrimaryTypes":["observed_problem","causal_claim"],"expectedReasoningModes":["investigate_contradiction","test_possible_explanations"],"shouldIdentify":["sales_decline","price_increase","temporal_correlation"],"shouldNotInfer":["price_increase_caused_the_fall"],"notes":"Paired with tc-005 — adds temporal anchor and proposed cause."}
+{"id":"tc-007","input":"The quarterly revenue exceeded targets but net profit declined by 12%.","expectedPrimaryTypes":["contradiction","unexplained_change"],"expectedReasoningModes":["investigate_contradiction","identify_missing_information"],"shouldIdentify":["revenue_above_target","profit_decline","divergence"],"shouldNotInfer":["cost_overrun_is_the_cause"],"notes":"Apparent contradiction — revenue up but profit down. Missing cost breakdown."}
+{"id":"tc-008","input":"Revenue from the premium tier dropped while total revenue grew.","expectedPrimaryTypes":["observed_problem","unexplained_change"],"expectedReasoningModes":["decompose_aggregate","identify_difference"],"shouldIdentify":["premium_tier_decline","total_revenue_growth","segment_cannibalization_risk"],"shouldNotInfer":["pricing_change_occurred"],"notes":"Aggregate masking — total growth hides segment decline."}
+{"id":"tc-009","input":"We need to improve our customer retention rate.","expectedPrimaryTypes":["decision_request","desired_outcome"],"expectedReasoningModes":["decision_support","identify_missing_information"],"shouldIdentify":["retention_improvement_desired","current_state_unknown"],"shouldNotInfer":["retention_rate_is_low","churn_has_increased"],"notes":"Desired outcome without stating the problem. Need to know if retention is actually bad."}
+{"id":"tc-010","input":"The system latency went from 200ms to 5 seconds on Tuesday.","expectedPrimaryTypes":["unexplained_change","observed_problem"],"expectedReasoningModes":["reconstruct_transition","identify_missing_information"],"shouldIdentify":["latency_baseline_200ms","latency_spike_5s","timestamp_tuesday"],"shouldNotInfer":["database_cause","release_cause"],"notes":"Specific measurement with timing anchor. Should identify transition but not infer cause."}
+{"id":"tc-011","input":"The new release should fix the login issue.","expectedPrimaryTypes":["decision_request","causal_claim"],"expectedReasoningModes":["validate_claim","investigate_contradiction"],"shouldIdentify":["proposed_solution","login_issue","solution_claim"],"shouldNotInfer":["login_issue_is_real","release_will_work"],"notes":"Proposed solution before problem is fully understood. Assumes the issue and fix are connected."}
+{"id":"tc-012","input":"I think therefore I am.","expectedPrimaryTypes":["ambiguous_statement","question"],"expectedReasoningModes":["clarify_meaning","identify_missing_information"],"shouldIdentify":["philosophical_statement","insufficient_operational_context"],"shouldNotInfer":["business_problem_exists","actionable_insight_possible"],"notes":"Ambiguous philosophical statement. Should not try to find operational meaning."}
+{"id":"tc-013","input":"I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.","expectedPrimaryTypes":["question","ambiguous_statement"],"expectedReasoningModes":["clarify_meaning"],"shouldIdentify":["meta_context","testing_hypothesis","self_reference"],"shouldNotInfer":[],"notes":"Paired with tc-12 — the meta-context changes classification entirely."}
+{"id":"tc-014","input":"The warehouse manager reported that inventory counts don't match the system.","expectedPrimaryTypes":["reported_claim","observed_problem"],"expectedReasoningModes":["validate_claim","investigate_contradiction"],"shouldIdentify":["warehouse_manager_report","inventory_mismatch","system_discrepancy","source_attribution"],"shouldNotInfer":["theft_occurred","software_bug"],"notes":"Reported claim — must distinguish what was said from what it means."}
+{"id":"tc-015","input":"We've seen a 35% increase in customer complaints.","expectedPrimaryTypes":["unexplained_change"],"expectedReasoningModes":["establish_baseline","validate_measurement"],"shouldIdentify":["complaints_increase","percentage_metric"],"shouldNotInfer":["product_quality_declined","customer_satisfaction_drop"],"notes":"Needs baseline — is this absolute or relative? Over what period?"}
+{"id":"tc-016","input":"The number of active users increased by 500%, from 4 to 2,001.","expectedPrimaryTypes":["unexplained_change"],"expectedReasoningModes":["validate_measurement","decompose_aggregate"],"shouldIdentify":["active_users_metric","absolute_vs_relative_growth","small_base_problem"],"shouldNotInfer":["product_success"],"notes":"Misleading absolute count where rate matters. Small base inflates percentage."}
+{"id":"tc-017","input":"User engagement metrics improved but the support ticket backlog grew by 200%.","expectedPrimaryTypes":["contradiction"],"expectedReasoningModes":["investigate_contradiction","identify_difference"],"shouldIdentify":["engagement_improvement","support_backlog_growth","divergent_metrics"],"shouldNotInfer":["users_are_angry","product_quality_is_worse"],"notes":"Two metrics telling opposite stories. Could mean engagement is superficial."}
+{"id":"tc-018","input":"The manufacturing team needs better quality control.","expectedPrimaryTypes":["decision_request","fault_report"],"expectedReasoningModes":["decision_support","identify_missing_information"],"shouldIdentify":["manufacturing_team","quality_control_desired"],"shouldNotInfer":["quality_is_bad","defect_rate_is_high"],"notes":"Solution proposed without problem specification. What specific quality issue?"}
+{"id":"tc-019","input":"All users in the EU region are getting a 403 error when trying to access the dashboard.","expectedPrimaryTypes":["observed_problem","fault_report"],"expectedReasoningModes":["validate_claim","identify_difference"],"shouldIdentify":["eu_region","403_error","access_denied","geographic_scope"],"shouldNotInfer":["gdpr_cause","regulatory_change"],"notes":"Geographic subset fault. Should not infer GDPR as cause without evidence."}
+{"id":"tc-020","input":"Some users in the EU region are getting a 403 error when trying to access the dashboard.","expectedPrimaryTypes":["observed_problem","fault_report"],"expectedReasoningModes":["validate_claim","identify_difference","decompose_aggregate"],"shouldIdentify":["eu_region_subset","403_error","partial_reachability"],"shouldNotInfer":["all_eu_users_affected"],"notes":"Paired with tc-19 — 'some' vs 'all' is the material difference."}
+{"id":"tc-021","input":"Production output was 1,200 units last month and 1,180 units this month.","expectedPrimaryTypes":["unexplained_change"],"expectedReasoningModes":["validate_measurement","establish_baseline"],"shouldIdentify":["production_output","month_over_month_decline","absolute_difference"],"shouldNotInfer":["efficiency_loss_occurred","equipment_failure"],"notes":"Small absolute change needs context — 1.7% drop might be normal variation."}
+{"id":"tc-022","input":"The CFO reported that the company's cash position is healthy.","expectedPrimaryTypes":["reported_claim"],"expectedReasoningModes":["validate_claim","identify_missing_information"],"shouldIdentify":["cfo_statement","cash_position_claim","source_attribution_cfo"],"shouldNotInfer":["cash_is_healthy","financial_stability_is_real"],"notes":"Reported opinion — must distinguish what was said from reality."}
+{"id":"tc-023","input":"We have enough funding to operate for 18 months.","expectedPrimaryTypes":["decision_request","observed_problem"],"expectedReasoningModes":["validate_claim","identify_missing_information"],"shouldIdentify":["funding_period","operational_sustainability","burn_rate_unknown"],"shouldNotInfer":["no_risk_exists"],"notes":"Claim about sustainability without burn rate context."}
+{"id":"tc-024","input":"The new feature was deployed at 3am and user complaints tripled the next day.","expectedPrimaryTypes":["causal_claim","unexplained_change"],"expectedReasoningModes":["test_possible_explanations","reconstruct_transition"],"shouldIdentify":["feature_deployment","timing_3am","complaint_tripling","temporal_relationship"],"shouldNotInfer":["deployment_caused_complaints"],"notes":"Temporal proximity ≠ causation. Should identify both events but not claim cause."}
+{"id":"tc-025","input":"We need to launch a mobile app to capture market share.","expectedPrimaryTypes":["decision_request","desired_outcome"],"expectedReasoningModes":["decision_support","identify_missing_information"],"shouldIdentify":["mobile_app_proposed","market_share_desired"],"shouldNotInfer":["no_mobile_app_exists","competitors_have_apps"],"notes":"Desired outcome without problem statement. What evidence supports this decision?"}
+{"id":"tc-026","input":"The system has been running for 90 days without failure since the migration.","expectedPrimaryTypes":["observed_problem"],"expectedReasoningModes":["validate_claim","establish_baseline"],"shouldIdentify":["uptime_90_days","post_migration_context","baseline_established"],"shouldNotInfer":["system_is_stable_forever"],"notes":"Positive claim about system stability with temporal anchor."}
+{"id":"tc-027","input":"No one has submitted the required compliance report despite multiple reminders.","expectedPrimaryTypes":["observed_problem","fault_report"],"expectedReasoningModes":["validate_claim","identify_missing_information","investigate_contradiction"],"shouldIdentify":["compliance_report","multiple_reminders","non_submission","absent_action"],"shouldNotInfer":["deliberate_refusal","negligence"],"notes":"Expected-but-missing information. Action was required but absent."}
+{"id":"tc-028","input":"The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.","expectedPrimaryTypes":["observed_problem","contradiction"],"expectedReasoningModes":["validate_measurement","decompose_aggregate"],"shouldIdentify":["audit_findings","incorrect_reports_rate_3_of_10","data_accuracy_issue"],"shouldNotInfer":["intentional_falsification","systemic_failure"],"notes":"Aggregate data — 30% error rate requires context about severity."}
+{"id":"tc-029","input":"We should implement the new CRM because our competitors have one.","expectedPrimaryTypes":["decision_request","causal_claim"],"expectedReasoningModes":["test_possible_explanations","validate_claim"],"shouldIdentify":["crm_proposal","competitor_comparison","competitive_pressure"],"shouldNotInfer":["crm_will_help","we_lack_crm","competitors_success_is_from_crm"],"notes":"FOMO-driven decision request without problem analysis."}
+{"id":"tc-030","input":"The server response time was acceptable last quarter but degraded this month.","expectedPrimaryTypes":["unexplained_change","observed_problem"],"expectedReasoningModes":["reconstruct_transition","identify_missing_information"],"shouldIdentify":["response_time_baseline_acceptable","degradation_timeline","quarter_to_month_comparison"],"shouldNotInfer":["load_increase_occurred"],"notes":"Baseline comparison with transition over time. Need specifics."}
+{"id":"tc-031","input":"The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.","expectedPrimaryTypes":["contradiction","observed_problem"],"expectedReasoningModes":["validate_claim","investigate_contradiction","identify_missing_information"],"shouldIdentify":["regulatory_requirement","data_location_violation","cross_border_backup"],"shouldNotInfer":["compliance_failure_is_certain"],"notes":"Regulatory conflict — requires verification of both claim and current state."}
+{"id":"tc-032","input":"External analysts expect our industry to decline by 15% next year due to regulatory changes.","expectedPrimaryTypes":["causal_claim","reported_claim"],"expectedReasoningModes":["validate_claim","test_possible_explanations"],"shouldIdentify":["industry_decline_prediction","external_source","regulatory_cause","15_percent_forecast"],"shouldNotInfer":["decline_will_occur"],"notes":"External prediction — must treat as claim, not fact."}
+{"id":"tc-033","input":"The database schema was changed on Friday but the reports are still working.","expectedPrimaryTypes":["unexplained_change"],"expectedReasoningModes":["validate_claim","test_possible_explanations"],"shouldIdentify":["schema_change","reports_working_post_change","unexpected_continuity"],"shouldNotInfer":["change_was_harmless"],"notes":"Expected impact did not occur — should flag as unexplained."}
+{"id":"tc-034","input":"Some team members say the new process is better while others say it's slower.","expectedPrimaryTypes":["contradiction","observed_problem"],"expectedReasoningModes":["validate_claim","investigate_contradiction","identify_missing_information"],"shouldIdentify":["subjective_split","new_process_evaluation","conflicting_opinions","measurement_gap"],"shouldNotInfer":["process_is_better_or_worse"],"notes":"Conflicting subjective claims — need measurable criteria."}
+{"id":"tc-035","input":"The application works fine on Chrome but not on Safari.","expectedPrimaryTypes":["observed_problem","fault_report"],"expectedReasoningModes":["validate_claim","identify_difference"],"shouldIdentify":["chrome_compatibility","safari_incompatibility","browser_specific_issue"],"shouldNotInfer":["webkit_bug"],"notes":"Browser-specific fault. Should identify the difference but not the technical cause."}