fix: resolve 500 errors from model returning trivial status objects (root cause + v0.2 prompt fix)
Two bugs were causing the model to return {"status":"ok"} / {"status":"ready"}
instead of structured reconstruction data, resulting in POST /api/analyse 500:
1. DOUBLE-WRAPPING BUG (lib/llm/provider.js):
generateReconstruction() called buildPrompt(scenario) on input that was
already a fully-built prompt string from analyseScenario(). This wrapped the
v0.1 prompt (~5000+ chars) in another template layer, producing incomprehensible
output that the model could not parse as structured JSON.
Fix: Pass scenario through directly (it is ALREADY a built prompt).
2. MISSING JSON SPEC (prompts/reconstruct-v0.2.md):
The v0.2 prompt template said 'matching the structure exactly' but never
defined what that structure was. The model invented its own field names
(input_classification, reasoning_mode, anchors) with snake_case instead of
camelCase, which failed Zod validation -> 500 errors.
Fix: Added explicit JSON schema section with exact key names, enum values,
and nested structure matching the Zod validation layer.
Additionally:
- Refactored route to use analyseScenario from lib/analysis (centralized)
- Added lib/analysis.js with shared analysis logic
- Updated components to display promptVersion and validation errors
- Added lib/reconstruction/prompt.js v0.1/v0.2 versioning
- Added lib/reconstruction/schema.js v0.2 Zod schemas
- Added debug tool scripts, evaluation results, and comparison findings
This commit is contained in:
+22
-78
@@ -1,16 +1,9 @@
|
||||
import { getConfig } from "@/lib/config";
|
||||
import { getProvider } from "@/lib/llm/provider";
|
||||
import { reconstructionSchema } from "@/lib/reconstruction/schema";
|
||||
|
||||
const MAX_SCENARIO_LENGTH = 10000;
|
||||
import { analyseScenario, PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION } from "@/lib/analysis";
|
||||
|
||||
export async function POST(request) {
|
||||
const startTime = Date.now();
|
||||
let rawResponse = null;
|
||||
|
||||
try {
|
||||
const body = await request.json();
|
||||
|
||||
|
||||
if (!body.scenario || typeof body.scenario !== "string") {
|
||||
return Response.json(
|
||||
{ error: "Request must include a 'scenario' string field" },
|
||||
@@ -18,83 +11,34 @@ export async function POST(request) {
|
||||
);
|
||||
}
|
||||
|
||||
const trimmed = body.scenario.trim();
|
||||
|
||||
if (trimmed.length === 0) {
|
||||
// Optional prompt version override
|
||||
let promptVersion = DEFAULT_PROMPT_VERSION;
|
||||
if (body.promptVersion && PROMPT_VERSIONS.includes(body.promptVersion)) {
|
||||
promptVersion = body.promptVersion;
|
||||
}
|
||||
|
||||
const result = await analyseScenario(body.scenario, { promptVersion });
|
||||
|
||||
if (!result.success) {
|
||||
return Response.json(
|
||||
{ error: "Scenario cannot be empty" },
|
||||
{ status: 400 }
|
||||
{ ...result, reconstruction: result.reconstruction || null },
|
||||
{ status: Number(result.statusCode) || 500 }
|
||||
);
|
||||
}
|
||||
|
||||
if (trimmed.length > MAX_SCENARIO_LENGTH) {
|
||||
return Response.json(
|
||||
{ error: `Scenario must be under ${MAX_SCENARIO_LENGTH} characters` },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const configResult = getConfig();
|
||||
if (!configResult.ok) {
|
||||
return Response.json(
|
||||
{ error: "Invalid server configuration" },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
|
||||
const { OLLAMA_BASE_URL, OLLAMA_MODEL } = configResult.config;
|
||||
const provider = getProvider();
|
||||
|
||||
// Attempt parse to capture raw for debugging
|
||||
let reconstruction;
|
||||
try {
|
||||
reconstruction = await provider.generateReconstruction(trimmed, OLLAMA_MODEL);
|
||||
} catch (e) {
|
||||
return Response.json(
|
||||
{
|
||||
error: e.message || "Unknown server error",
|
||||
responseDurationMs: Date.now() - startTime,
|
||||
modelName: OLLAMA_MODEL,
|
||||
validationStatus: "invalid",
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
|
||||
// Try to stringify for rawResponse display (safe even if it's already an object)
|
||||
try {
|
||||
rawResponse = JSON.stringify(reconstruction);
|
||||
} catch {
|
||||
rawResponse = String(reconstruction).slice(0, 2000);
|
||||
}
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
// Validate with Zod schema
|
||||
const validationResult = reconstructionSchema.safeParse(reconstruction);
|
||||
|
||||
if (!validationResult.success) {
|
||||
return Response.json({
|
||||
reconstruction: null,
|
||||
modelName: OLLAMA_MODEL,
|
||||
responseDurationMs: duration,
|
||||
validationStatus: "invalid",
|
||||
rawResponse: rawResponse?.slice(0, 2000),
|
||||
errors: validationResult.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`),
|
||||
});
|
||||
}
|
||||
|
||||
return Response.json({
|
||||
reconstruction: validationResult.data,
|
||||
modelName: OLLAMA_MODEL,
|
||||
responseDurationMs: duration,
|
||||
validationStatus: "valid",
|
||||
rawResponse: rawResponse?.slice(0, 2000),
|
||||
inputClassification: result.inputClassification,
|
||||
reconstruction: result.reconstruction,
|
||||
evidence: result.evidence,
|
||||
nextQuestion: result.nextQuestion,
|
||||
modelName: result.modelName,
|
||||
responseDurationMs: result.responseDurationMs,
|
||||
validationStatus: result.validationStatus,
|
||||
promptVersion: result.promptVersion,
|
||||
});
|
||||
} catch (e) {
|
||||
const duration = Date.now() - startTime;
|
||||
return Response.json(
|
||||
{ error: e.message || "Unknown server error", responseDurationMs: duration },
|
||||
{ error: e.message || "Unknown server error", responseDurationMs: 0 },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -16,9 +16,19 @@ const ValidationIndicator = ({ status }) => {
|
||||
);
|
||||
};
|
||||
|
||||
const validationIcons = {
|
||||
valid: "✅",
|
||||
partial: "⚠️",
|
||||
invalid: "❌",
|
||||
};
|
||||
|
||||
export default function DiagnosticsView({ result }) {
|
||||
if (!result) return null;
|
||||
|
||||
const metrics = [
|
||||
{ label: "Model", value: result.modelName || "?" },
|
||||
{ label: "Provider", value: "Ollama" },
|
||||
{ label: "Prompt version", value: result.promptVersion || "?" },
|
||||
{ label: "Duration", value: result.responseDurationMs != null ? `${result.responseDurationMs}ms` : "?" },
|
||||
{ label: "Validation", value: <ValidationIndicator status={result.validationStatus || "invalid"} /> },
|
||||
];
|
||||
@@ -35,16 +45,31 @@ export default function DiagnosticsView({ result }) {
|
||||
))}
|
||||
</dl>
|
||||
|
||||
{/* Collapsed raw output for debugging */}
|
||||
{result.rawResponse && (
|
||||
<details className="mt-4">
|
||||
<summary className="cursor-pointer text-xs text-gray-500 underline hover:text-gray-700">
|
||||
View raw model response
|
||||
View raw model response ({(result.rawResponse?.length || 0).toLocaleString()} chars)
|
||||
</summary>
|
||||
<pre className="mt-2 max-h-60 overflow-auto rounded bg-gray-900 px-3 py-2 text-xs leading-relaxed text-green-400">
|
||||
{result.rawResponse}
|
||||
</pre>
|
||||
</details>
|
||||
)}
|
||||
|
||||
{/* Errors if present */}
|
||||
{result.errors && result.errors.length > 0 && (
|
||||
<details className="mt-3">
|
||||
<summary className="cursor-pointer text-xs text-red-500 underline hover:text-red-700">
|
||||
Validation errors ({result.errors.length})
|
||||
</summary>
|
||||
<ul className="mt-1 space-y-0.5 text-xs text-red-600">
|
||||
{result.errors.map((err, i) => (
|
||||
<li key={i}>{err}</li>
|
||||
))}
|
||||
</ul>
|
||||
</details>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,15 +1,8 @@
|
||||
const categoryLabels = {
|
||||
observations: "Direct Observations",
|
||||
reportedClaims: "Reported Claims",
|
||||
assumptions: "Unsupported Assumptions",
|
||||
entities: "Entities",
|
||||
transitions: "Transitions",
|
||||
expectedButMissing: "Expected But Missing",
|
||||
presentButUnexpected: "Present But Unexpected",
|
||||
contradictions: "Contradictions",
|
||||
openUncertainties: "Open Uncertainties",
|
||||
};
|
||||
"use client";
|
||||
|
||||
import { useMemo } from "react";
|
||||
|
||||
// ── Confidence badge (shared) ────────────────────────
|
||||
const confidenceColor = {
|
||||
low: "text-red-600 bg-red-50 border-red-200",
|
||||
medium: "text-yellow-700 bg-yellow-50 border-yellow-200",
|
||||
@@ -22,26 +15,210 @@ const ConfidenceBadge = ({ level }) => (
|
||||
</span>
|
||||
);
|
||||
|
||||
function ItemList({ items, renderExtra }) {
|
||||
if (!items?.length) return <p className="text-sm italic text-gray-400">None identified</p>;
|
||||
|
||||
// ── Evidence type labels (shared) ───────────────────
|
||||
const evidenceTypeLabels = {
|
||||
direct_observation: "Direct Observation",
|
||||
reported_statement: "Reported Statement",
|
||||
interpretation: "Interpretation",
|
||||
assumption: "Assumption",
|
||||
inferred_relationship: "Inferred Relationship",
|
||||
};
|
||||
|
||||
const importanceColors = {
|
||||
incidental: "text-gray-500 bg-gray-50 border-gray-200",
|
||||
supporting: "text-blue-700 bg-blue-50 border-blue-200",
|
||||
important: "text-orange-700 bg-orange-50 border-orange-200",
|
||||
critical: "text-red-800 bg-red-50 border-red-300 font-semibold",
|
||||
};
|
||||
|
||||
const importanceLabels = {
|
||||
incidental: "Incidental",
|
||||
supporting: "Supporting",
|
||||
important: "Important",
|
||||
critical: "Critical",
|
||||
};
|
||||
|
||||
// ── Input classification display ────────────────────
|
||||
function ClassificationDisplay({ classification }) {
|
||||
if (!classification) return null;
|
||||
const p = classification.primaryType || classification.primary_type;
|
||||
const sec = classification.secondaryTypes || classification.secondary_types || [];
|
||||
const modes = classification.reasoningModes || classification.reasoning_modes || [];
|
||||
|
||||
// Normalize camelCase to snake_case for display if needed
|
||||
const primaryLabel = String(p).replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
|
||||
const secLabels = sec.map((s) => s.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()));
|
||||
const modeLabels = modes.map((m) => m.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()));
|
||||
|
||||
return (
|
||||
<ul className="space-y-2">
|
||||
{items.map((item) => (
|
||||
<li key={item.id} className="rounded border border-gray-200 bg-white px-3 py-2 text-sm">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="font-mono text-xs text-gray-400">#{item.id}</span>
|
||||
<ConfidenceBadge level={item.confidence} />
|
||||
</div>
|
||||
<p className="mt-1">{item.description}</p>
|
||||
{renderExtra && renderExtra(item)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
<div className="rounded-lg border border-blue-200 bg-blue-50 p-4">
|
||||
<h3 className="mb-2 text-sm font-semibold text-blue-700">Input Classification</h3>
|
||||
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1.5 text-sm">
|
||||
<dt className="text-blue-500">Primary type</dt>
|
||||
<dd className="font-medium">{primaryLabel}</dd>
|
||||
{secLabels.length > 0 && (
|
||||
<>
|
||||
<dt className="text-blue-500 pt-1">Secondary types</dt>
|
||||
<dd>{secLabels.join(" · ")}</dd>
|
||||
</>
|
||||
)}
|
||||
{modeLabels.length > 0 && (
|
||||
<>
|
||||
<dt className="text-blue-500 pt-1">Reasoning modes</dt>
|
||||
<dd>{modeLabels.join(" · ")}</dd>
|
||||
</>
|
||||
)}
|
||||
<dt className="text-blue-500 pt-1">Classification reason</dt>
|
||||
<dd className="italic">{classification.classificationReason || classification.classification_reason}</dd>
|
||||
<dt className="text-blue-500 pt-1">Confidence</dt>
|
||||
<dd><ConfidenceBadge level={classification.confidence} /></dd>
|
||||
</dl>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Reconstruction summary ──────────────────────────
|
||||
function SummaryDisplay({ reconstruction }) {
|
||||
if (!reconstruction?.summary) return null;
|
||||
const summary = reconstruction.summary || reconstruction.Summary;
|
||||
return (
|
||||
<div className="rounded-lg border border-gray-200 bg-white p-4">
|
||||
<h3 className="mb-2 text-sm font-semibold text-gray-600">Reconstruction Summary</h3>
|
||||
<p className="text-sm leading-relaxed">{summary}</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Generic item list (used for multiple sections) ──
|
||||
function ItemList({ title, items, renderExtra }) {
|
||||
const count = items?.length;
|
||||
if (!count) return null; // hide empty sections entirely
|
||||
|
||||
const itemsArr = Array.isArray(items) ? items : [items];
|
||||
|
||||
return (
|
||||
<div className="mb-4 rounded-lg border border-gray-200 bg-white p-4">
|
||||
<h3 className="mb-2 text-sm font-semibold text-gray-600">{title} ({count})</h3>
|
||||
<ul className="space-y-2">
|
||||
{itemsArr.map((item, idx) => (
|
||||
<li key={item.id || `${title}-${idx}`} className="rounded border border-gray-200 bg-white px-3 py-2 text-sm">
|
||||
<div className="flex items-center gap-2">
|
||||
{item.id && <span className="font-mono text-xs text-gray-400">#{item.id}</span>}
|
||||
{item.confidence && <ConfidenceBadge level={item.confidence} />}
|
||||
{item.importance && (
|
||||
<span className={`inline-block rounded-full border px-2 py-0.5 text-xs font-medium ${importanceColors[item.importance] || "text-gray-600 bg-gray-100"}`}>
|
||||
{importanceLabels[item.importance]}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<p className="mt-1">{item.description}</p>
|
||||
{renderExtra && renderExtra(item)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Plausible interpretations ───────────────────────
|
||||
function InterpretationsDisplay({ interpretations }) {
|
||||
if (!interpretations?.length) return null;
|
||||
const arr = Array.isArray(interpretations) ? interpretations : [interpretations];
|
||||
|
||||
return (
|
||||
<div className="mb-4 rounded-lg border border-indigo-200 bg-indigo-50 p-4">
|
||||
<h3 className="mb-2 text-sm font-semibold text-indigo-700">Plausible Interpretations ({arr.length})</h3>
|
||||
<ul className="space-y-3">
|
||||
{arr.map((interp, idx) => (
|
||||
<li key={interp.id || `${idx}`} className="rounded border border-indigo-200 bg-white px-3 py-2.5 text-sm leading-relaxed">
|
||||
<div className="flex items-center gap-2 mb-1">
|
||||
<span className="font-medium text-indigo-600">{interp.description}</span>
|
||||
{interp.confidence && <ConfidenceBadge level={interp.confidence} />}
|
||||
</div>
|
||||
{interp.supportingEvidenceIds?.length > 0 && (
|
||||
<p className="text-xs text-gray-500">Supporting evidence: {interp.supportingEvidenceIds.join(", ")}</p>
|
||||
)}
|
||||
{interp.assumptionsRequired?.length > 0 && (
|
||||
<p className="text-xs italic text-gray-500">Requires assumptions: {interp.assumptionsRequired.join("; ")}</p>
|
||||
)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Next question (prominent) ───────────────────────
|
||||
function NextQuestionDisplay({ question }) {
|
||||
if (!question?.question) return null;
|
||||
const q = question.question || question.Question;
|
||||
const targets = question.targets || question.Targets || [];
|
||||
const reason = question.reason || question.Reason || "";
|
||||
const value = question.expectedInformationValue || question.expected_information_value || "medium";
|
||||
|
||||
const valueLabel = { low: "Low", medium: "Medium", high: "High" }[value] || "Medium";
|
||||
const valueColor = { low: "bg-yellow-100 text-yellow-800", medium: "bg-blue-100 text-blue-800", high: "bg-green-100 text-green-800" }[value] || "";
|
||||
|
||||
return (
|
||||
<div className="rounded-lg border-2 border-green-300 bg-green-50 p-5">
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
<h3 className="text-sm font-bold text-green-800">Next Question</h3>
|
||||
<span className={`rounded-full px-2 py-0.5 text-xs font-medium ${valueColor}`}>{valueLabel} value</span>
|
||||
</div>
|
||||
<p className="mb-2 text-base font-medium text-gray-900">{q}</p>
|
||||
{targets.length > 0 && (
|
||||
<p className="text-sm text-gray-600">Targets: {targets.join(", ")}</p>
|
||||
)}
|
||||
{reason && <p className="text-sm italic text-gray-500">Because: {reason}</p>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Evidence list ───────────────────────────────────
|
||||
function EvidenceDisplay({ evidence }) {
|
||||
if (!evidence?.length) return null;
|
||||
const arr = Array.isArray(evidence) ? evidence : [evidence];
|
||||
|
||||
const evidenceLabels = {
|
||||
direct_observation: "👁 Direct Observation",
|
||||
reported_statement: "🗣 Reported Statement",
|
||||
interpretation: "💡 Interpretation",
|
||||
assumption: "❓ Assumption",
|
||||
inferred_relationship: "🔗 Inferred Relationship",
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="mb-4 rounded-lg border border-gray-200 bg-white p-4">
|
||||
<h3 className="mb-2 text-sm font-semibold text-gray-600">Supporting Evidence ({arr.length})</h3>
|
||||
<ul className="space-y-2">
|
||||
{arr.map((item, idx) => (
|
||||
<li key={item.id || `${idx}`} className="rounded border border-gray-200 bg-white px-3 py-2 text-sm leading-relaxed">
|
||||
<div className="flex items-center gap-2 mb-0.5 flex-wrap">
|
||||
{item.id && <span className="font-mono text-xs text-gray-400">#{item.id}</span>}
|
||||
<span className={`inline-block rounded px-1.5 py-0.5 text-[10px] font-medium ${importanceColors[item.importance] || "text-gray-600 bg-gray-100"}`}>
|
||||
{importanceLabels[item.importance]}
|
||||
</span>
|
||||
<span className="inline-block rounded px-1.5 py-0.5 text-[10px] font-medium bg-gray-100 text-gray-700">
|
||||
{evidenceLabels[item.evidenceType] || item.evidenceType}
|
||||
</span>
|
||||
{item.confidence && <ConfidenceBadge level={item.confidence} />}
|
||||
</div>
|
||||
<p className="text-sm">{item.description}</p>
|
||||
{(item.source || item.attribution) && (
|
||||
<p className="mt-0.5 text-xs text-gray-400">Source: {item.source || item.attribution}</p>
|
||||
)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Main component ──────────────────────────────────
|
||||
export default function ReconstructionView({ reconstruction, partial }) {
|
||||
// Handle both v0.2 direct object and wrapped result formats
|
||||
const data = reconstruction;
|
||||
|
||||
if (partial) {
|
||||
return (
|
||||
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-3 text-sm text-yellow-800">
|
||||
@@ -50,21 +227,74 @@ export default function ReconstructionView({ reconstruction, partial }) {
|
||||
);
|
||||
}
|
||||
|
||||
const categories = Object.entries(categoryLabels).map(([key, label]) => ({
|
||||
key,
|
||||
label,
|
||||
items: reconstruction[key],
|
||||
}));
|
||||
|
||||
return (
|
||||
<div className="space-y-1">
|
||||
<h2 className="mb-3 text-lg font-semibold">Reconstruction</h2>
|
||||
{categories.map(({ key, label, items }) => (
|
||||
<div key={key} className="mb-4 rounded border border-gray-200 bg-white p-4">
|
||||
<h3 className="mb-2 text-sm font-medium text-gray-600">{label}</h3>
|
||||
<ItemList items={items} />
|
||||
</div>
|
||||
))}
|
||||
<div className="space-y-4">
|
||||
{/* Classification first */}
|
||||
{data.inputClassification && (
|
||||
<ClassificationDisplay classification={data.inputClassification} />
|
||||
)}
|
||||
|
||||
{/* Summary */}
|
||||
{data.reconstruction?.summary && (
|
||||
<SummaryDisplay reconstruction={data.reconstruction} />
|
||||
)}
|
||||
|
||||
{/* Key differences */}
|
||||
{data.reconstruction?.differences && (
|
||||
<ItemList title="Key Differences" items={data.reconstruction.differences} />
|
||||
)}
|
||||
|
||||
{/* Unexplained transitions */}
|
||||
{data.reconstruction?.unexplainedTransitions && data.reconstruction.unexplainedTransitions.length > 0 && (
|
||||
<ItemList title="Unexplained Transitions" items={data.reconstruction.unexplainedTransitions} renderExtra={(i) => (
|
||||
i.entity && <p className="mt-1 text-xs text-gray-500">Entity: {i.entity}</p>
|
||||
)} />
|
||||
)}
|
||||
|
||||
{/* Contradictions */}
|
||||
{data.reconstruction?.contradictions && data.reconstruction.contradictions.length > 0 && (
|
||||
<ItemList title="Contradictions" items={data.reconstruction.contradictions} />
|
||||
)}
|
||||
|
||||
{/* Important unknowns */}
|
||||
{data.reconstruction?.importantUnknowns && data.reconstruction.importantUnknowns.length > 0 && (
|
||||
<ItemList title="Important Unknowns" items={data.reconstruction.importantUnknowns} />
|
||||
)}
|
||||
|
||||
{/* Plausible interpretations */}
|
||||
{data.reconstruction?.plausibleInterpretations && data.reconstruction.plausibleInterpretations.length > 0 && (
|
||||
<InterpretationsDisplay interpretations={data.reconstruction.plausibleInterpretations} />
|
||||
)}
|
||||
|
||||
{/* Secondary reconstruction categories (actors, systems, etc.) */}
|
||||
{data.reconstruction?.actors && data.reconstruction.actors.length > 0 && (
|
||||
<ItemList title="Actors" items={data.reconstruction.actors} />
|
||||
)}
|
||||
{data.reconstruction?.systemsOrObjects && data.reconstruction.systemsOrObjects.length > 0 && (
|
||||
<ItemList title="Systems / Objects" items={data.reconstruction.systemsOrObjects} />
|
||||
)}
|
||||
{data.reconstruction?.expectedStates && data.reconstruction.expectedStates.length > 0 && (
|
||||
<ItemList title="Expected States" items={data.reconstruction.expectedStates} />
|
||||
)}
|
||||
{data.reconstruction?.observedStates && data.reconstruction.observedStates.length > 0 && (
|
||||
<ItemList title="Observed States" items={data.reconstruction.observedStates} />
|
||||
)}
|
||||
{data.reconstruction?.knownTransitions && data.reconstruction.knownTransitions.length > 0 && (
|
||||
<ItemList title="Known Transitions" items={data.reconstruction.knownTransitions} renderExtra={(i) => (
|
||||
<div className="mt-1 text-xs text-gray-500">
|
||||
{i.entity && <span>Entity: {i.entity} · </span>}
|
||||
From "{i.previousState}" → To "{i.currentState}" ({i.explanationStatus})
|
||||
</div>
|
||||
)} />
|
||||
)}
|
||||
|
||||
{/* Next question — prominent */}
|
||||
<NextQuestionDisplay question={data.nextQuestion} />
|
||||
|
||||
{/* Evidence */}
|
||||
{data.evidence && (
|
||||
<EvidenceDisplay evidence={data.evidence} />
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ const MAX_LENGTH = 10000;
|
||||
|
||||
export default function ScenarioForm() {
|
||||
const [scenario, setScenario] = useState("");
|
||||
const [status, setStatus] = useState("idle"); // idle | loading | error | success
|
||||
const [status, setStatus] = useState("idle"); // idle | loading | error | success | partial
|
||||
const [result, setResult] = useState(null);
|
||||
const textareaRef = useRef(null);
|
||||
|
||||
@@ -29,6 +29,10 @@ export default function ScenarioForm() {
|
||||
if (res.ok && data.validationStatus === "valid") {
|
||||
setStatus("success");
|
||||
setResult(data);
|
||||
} else if (data.success) {
|
||||
// Success in analysis but validation may be partial
|
||||
setStatus("success");
|
||||
setResult(data);
|
||||
} else {
|
||||
setStatus("error");
|
||||
setResult(data);
|
||||
@@ -39,8 +43,12 @@ export default function ScenarioForm() {
|
||||
}
|
||||
};
|
||||
|
||||
// Always show diagnostics when there's a result (even if validation failed)
|
||||
const hasDiagnostics = result && (result.reconstruction || result.modelName || result.responseDurationMs !== undefined);
|
||||
// Determine if we have meaningful content to display
|
||||
const hasClassification = result?.inputClassification;
|
||||
const hasReconstruction = result?.reconstruction;
|
||||
const hasNextQuestion = result?.nextQuestion;
|
||||
const hasEvidence = result?.evidence && result.evidence.length > 0;
|
||||
const hasMeaningfulContent = hasClassification || hasReconstruction || hasNextQuestion || hasEvidence;
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
@@ -65,6 +73,7 @@ export default function ScenarioForm() {
|
||||
</div>
|
||||
</form>
|
||||
|
||||
{/* Error state */}
|
||||
{status === "error" && (
|
||||
<div className="space-y-3">
|
||||
{result?.error && (
|
||||
@@ -72,36 +81,47 @@ export default function ScenarioForm() {
|
||||
Error: {result.error}
|
||||
</div>
|
||||
)}
|
||||
{hasDiagnostics && result?.modelName && (
|
||||
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1.5 text-sm">
|
||||
<dt className="text-gray-500">Model</dt>
|
||||
<dd>{result.modelName}</dd>
|
||||
<dt className="text-gray-500">Duration</dt>
|
||||
<dd>{result.responseDurationMs != null ? `${result.responseDurationMs}ms` : "?"}</dd>
|
||||
</dl>
|
||||
{/* Show partial content even on validation failure */}
|
||||
{(hasClassification || hasReconstruction) && (
|
||||
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
|
||||
⚠ Partial result — some fields failed validation. Showing what was accepted.
|
||||
</div>
|
||||
)}
|
||||
{hasReconstruction && (
|
||||
<ReconstructionView reconstruction={result} partial />
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{status === "success" && result?.reconstruction && (
|
||||
{/* Success state */}
|
||||
{status === "success" && hasMeaningfulContent && (
|
||||
<div className="space-y-4">
|
||||
<ReconstructionView reconstruction={result.reconstruction} />
|
||||
<DiagnosticsView result={result} />
|
||||
<ReconstructionView reconstruction={result} />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{status === "error" && result?.reconstruction && (
|
||||
<div className="space-y-3">
|
||||
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
|
||||
⚠ Partial result — some fields failed validation. Showing what was accepted.
|
||||
</div>
|
||||
<ReconstructionView reconstruction={result.reconstruction} partial />
|
||||
</div>
|
||||
{/* Always show diagnostics when we have any result */}
|
||||
{(hasClassification || hasReconstruction || hasNextQuestion) && (
|
||||
<DiagnosticsView result={result} />
|
||||
)}
|
||||
|
||||
{status === "loading" && (
|
||||
<div className="py-12 text-center text-sm text-gray-400">Waiting for model response...</div>
|
||||
)}
|
||||
|
||||
{/* Empty state */}
|
||||
{status === "idle" && (
|
||||
<div className="rounded-lg border border-dashed border-gray-300 bg-gray-50 px-6 py-8 text-center">
|
||||
<p className="text-sm text-gray-400">Enter a scenario above and click Analyse to begin.</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Invalid result with no partial data */}
|
||||
{status === "error" && !result?.error && !hasMeaningfulContent && (
|
||||
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
|
||||
Validation failed — no structured output was produced.
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"id": "diag-01",
|
||||
"description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.",
|
||||
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
||||
"responseDurationMs": 6,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "complaints",
|
||||
"found": true
|
||||
},
|
||||
{
|
||||
"concept": "warehouse",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "baseline comparison",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "quality issue",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "staff turnover",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "training gap",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
# Diagnostic Case: diag-01
|
||||
|
||||
Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
We've seen a spike in complaints from our warehouse team this month compared to last month.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ✅ PASS (1/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: unexplained_change
|
||||
- **Actual Reasoning Modes**: establish_baseline, identify_difference
|
||||
- **Response Duration**: 6ms
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): warehouse, baseline comparison
|
||||
@@ -0,0 +1,55 @@
|
||||
{
|
||||
"id": "diag-02",
|
||||
"description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.",
|
||||
"input": "Some customers reported that the new app crashes when uploading photos.",
|
||||
"responseDurationMs": 2,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "app crashes",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "photo upload",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "some customers",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "all users affected",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "server-side bug",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "Android only",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
# Diagnostic Case: diag-02
|
||||
|
||||
Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Some customers reported that the new app crashes when uploading photos.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ✅ PASS (1/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: observed_problem
|
||||
- **Actual Reasoning Modes**: identify_difference
|
||||
- **Response Duration**: 2ms
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): app crashes, photo upload, some customers
|
||||
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"id": "diag-03",
|
||||
"description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.",
|
||||
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": false,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "sales decline",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "price increase",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "revenue increase",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "CFO report",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "price was set too high",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "competitors gained market share",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "revenue data is wrong",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-03
|
||||
|
||||
Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: causal_claim
|
||||
- **Actual Reasoning Modes**: establish_baseline, identify_difference
|
||||
- **Response Duration**: 1ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
classification mismatch
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): sales decline, price increase, revenue increase, CFO report
|
||||
@@ -0,0 +1,57 @@
|
||||
{
|
||||
"id": "diag-04",
|
||||
"description": "Decision request — forward-looking, needs missing info identification.",
|
||||
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "marketplace app",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "Southeast Asia",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "competitor gap",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "this will definitely succeed",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "we have the resources",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "competitors are struggling",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
# Diagnostic Case: diag-04
|
||||
|
||||
Decision request — forward-looking, needs missing info identification.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ✅ PASS (1/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: decision_request
|
||||
- **Actual Reasoning Modes**: identify_difference, decision_support, identify_missing_information
|
||||
- **Response Duration**: 0ms
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): marketplace app, Southeast Asia, competitor gap
|
||||
@@ -0,0 +1,57 @@
|
||||
{
|
||||
"id": "diag-05",
|
||||
"description": "Unexpected continuity — changed context but no outcome change.",
|
||||
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "supplier change",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "three months ago",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "same defect rate",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "new supplier is worse",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "old supplier was better",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "quality process is broken",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
# Diagnostic Case: diag-05
|
||||
|
||||
Unexpected continuity — changed context but no outcome change.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Our production line changed suppliers three months ago but still delivers the same defect rate as before.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ✅ PASS (1/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: unexplained_change
|
||||
- **Actual Reasoning Modes**: identify_difference, establish_baseline, validate_measurement
|
||||
- **Response Duration**: 1ms
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): supplier change, three months ago, same defect rate
|
||||
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"id": "diag-06",
|
||||
"description": "Quantified improvement — needs context about measurement period and baseline conditions.",
|
||||
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": false,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "completion rate",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "45%",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "62%",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "onboarding",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "all improvements are due to the redesign",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "the old flow was bad",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "users prefer the new design",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-06
|
||||
|
||||
Quantified improvement — needs context about measurement period and baseline conditions.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
From 45% to 62%, the completion rate for our onboarding flow improved significantly.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: observed_problem
|
||||
- **Actual Reasoning Modes**: identify_difference
|
||||
- **Response Duration**: 1ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
classification mismatch
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): completion rate, 45%, 62%, onboarding
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"id": "diag-07",
|
||||
"description": "Single reported claim — needs validation, not acceptance as fact.",
|
||||
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "pricing complexity",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "small business",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "user claim",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "the pricing is actually complex",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "other small businesses agree",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "we should simplify pricing",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
# Diagnostic Case: diag-07
|
||||
|
||||
Single reported claim — needs validation, not acceptance as fact.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
A user claimed that our pricing model is too complex for small businesses.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ✅ PASS (1/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: reported_claim
|
||||
- **Actual Reasoning Modes**: identify_difference, validate_claim
|
||||
- **Response Duration**: 1ms
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): pricing complexity, small business, user claim
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"id": "diag-08",
|
||||
"description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode.",
|
||||
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "philosophical",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "ambiguous",
|
||||
"found": true
|
||||
},
|
||||
{
|
||||
"concept": "meaning clarification",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "the phrase was wrong",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "the colleague is hostile",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "we should avoid philosophical language",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
# Diagnostic Case: diag-08
|
||||
|
||||
Meta-test — self-referential ambiguous statement. Should trigger clarification mode.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ✅ PASS (1/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: ambiguous_statement
|
||||
- **Actual Reasoning Modes**: identify_difference, clarify_meaning
|
||||
- **Response Duration**: 1ms
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): philosophical, meaning clarification
|
||||
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"id": "diag-09",
|
||||
"description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.",
|
||||
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "deployment",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "complaint volume increase",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "tripled",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "47 cases",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "the deployment caused the complaints",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "the bug report was insufficient",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "rollback is needed",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
# Diagnostic Case: diag-09
|
||||
|
||||
Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
After the deployment last week, our complaint volume tripled to 47 cases per day.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ✅ PASS (1/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: causal_claim
|
||||
- **Actual Reasoning Modes**: establish_baseline, identify_difference
|
||||
- **Response Duration**: 1ms
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): deployment, complaint volume increase, tripled, 47 cases
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"id": "diag-10",
|
||||
"description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims.",
|
||||
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "production issues",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "delivery speed",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "complaint types",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "production is worse than delivery",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "the delivery team needs training",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "both teams are underperforming equally",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
# Diagnostic Case: diag-10
|
||||
|
||||
Paired with diag-01 — distinguishes subset complaints from aggregate claims.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Some complaints involve production issues, but others say the delivery team is slow.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ✅ PASS (1/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: observed_problem
|
||||
- **Actual Reasoning Modes**: establish_baseline, identify_difference
|
||||
- **Response Duration**: 1ms
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
missing required concept(s): production issues, delivery speed, complaint types
|
||||
@@ -0,0 +1,585 @@
|
||||
{
|
||||
"timestamp": "2026-08-01T06:01:56.070Z",
|
||||
"provider": "mock",
|
||||
"promptVersion": "v0.2",
|
||||
"casesRun": 10,
|
||||
"summary": {
|
||||
"technical": {
|
||||
"schemaValidityRate": "100.0%",
|
||||
"classificationMatchRate": "80.0%",
|
||||
"nextQuestionPresentRate": "100.0%",
|
||||
"passRate": "80.0%"
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConceptMatchRate": "0.0%",
|
||||
"unsupportedInferenceFailures": "0",
|
||||
"passRate": "0.0%"
|
||||
},
|
||||
"combinedPassRate": "0.0%",
|
||||
"averageResponseDurationMs": "2"
|
||||
},
|
||||
"testCaseResults": [
|
||||
{
|
||||
"id": "diag-01",
|
||||
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
||||
"responseDurationMs": 6,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "complaints",
|
||||
"found": true
|
||||
},
|
||||
{
|
||||
"concept": "warehouse",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "baseline comparison",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "quality issue",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "staff turnover",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "training gap",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-02",
|
||||
"input": "Some customers reported that the new app crashes when uploading photos.",
|
||||
"responseDurationMs": 2,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "app crashes",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "photo upload",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "some customers",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "all users affected",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "server-side bug",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "Android only",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-03",
|
||||
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": false,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "sales decline",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "price increase",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "revenue increase",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "CFO report",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "price was set too high",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "competitors gained market share",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "revenue data is wrong",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-04",
|
||||
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "marketplace app",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "Southeast Asia",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "competitor gap",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "this will definitely succeed",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "we have the resources",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "competitors are struggling",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-05",
|
||||
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "supplier change",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "three months ago",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "same defect rate",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "new supplier is worse",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "old supplier was better",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "quality process is broken",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-06",
|
||||
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": false,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "completion rate",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "45%",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "62%",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "onboarding",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "all improvements are due to the redesign",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "the old flow was bad",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "users prefer the new design",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-07",
|
||||
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "pricing complexity",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "small business",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "user claim",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "the pricing is actually complex",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "other small businesses agree",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "we should simplify pricing",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-08",
|
||||
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "philosophical",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "ambiguous",
|
||||
"found": true
|
||||
},
|
||||
{
|
||||
"concept": "meaning clarification",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "the phrase was wrong",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "the colleague is hostile",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "we should avoid philosophical language",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-09",
|
||||
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "deployment",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "complaint volume increase",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "tripled",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "47 cases",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "the deployment caused the complaints",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "the bug report was insufficient",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "rollback is needed",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-10",
|
||||
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"technical": {
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"nextQuestionPresent": true,
|
||||
"pass": true,
|
||||
"errors": []
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": false,
|
||||
"details": [
|
||||
{
|
||||
"concept": "production issues",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "delivery speed",
|
||||
"found": false
|
||||
},
|
||||
{
|
||||
"concept": "complaint types",
|
||||
"found": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": [
|
||||
{
|
||||
"concept": "production is worse than delivery",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "the delivery team needs training",
|
||||
"absent": true
|
||||
},
|
||||
{
|
||||
"concept": "both teams are underperforming equally",
|
||||
"absent": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"id": "diag-01",
|
||||
"description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.",
|
||||
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
||||
"responseDurationMs": 1785564462515,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-01
|
||||
|
||||
Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
We've seen a spike in complaints from our warehouse team this month compared to last month.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462515ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"id": "diag-02",
|
||||
"description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.",
|
||||
"input": "Some customers reported that the new app crashes when uploading photos.",
|
||||
"responseDurationMs": 1785564462519,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-02
|
||||
|
||||
Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Some customers reported that the new app crashes when uploading photos.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462519ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"id": "diag-03",
|
||||
"description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.",
|
||||
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
||||
"responseDurationMs": 1785564462519,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-03
|
||||
|
||||
Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462519ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"id": "diag-04",
|
||||
"description": "Decision request — forward-looking, needs missing info identification.",
|
||||
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
||||
"responseDurationMs": 1785564462520,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-04
|
||||
|
||||
Decision request — forward-looking, needs missing info identification.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462520ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"id": "diag-05",
|
||||
"description": "Unexpected continuity — changed context but no outcome change.",
|
||||
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
||||
"responseDurationMs": 1785564462520,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-05
|
||||
|
||||
Unexpected continuity — changed context but no outcome change.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Our production line changed suppliers three months ago but still delivers the same defect rate as before.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462520ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"id": "diag-06",
|
||||
"description": "Quantified improvement — needs context about measurement period and baseline conditions.",
|
||||
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
||||
"responseDurationMs": 1785564462521,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-06
|
||||
|
||||
Quantified improvement — needs context about measurement period and baseline conditions.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
From 45% to 62%, the completion rate for our onboarding flow improved significantly.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462521ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"id": "diag-07",
|
||||
"description": "Single reported claim — needs validation, not acceptance as fact.",
|
||||
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
||||
"responseDurationMs": 1785564462521,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-07
|
||||
|
||||
Single reported claim — needs validation, not acceptance as fact.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
A user claimed that our pricing model is too complex for small businesses.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462521ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"id": "diag-08",
|
||||
"description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode.",
|
||||
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
||||
"responseDurationMs": 1785564462521,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-08
|
||||
|
||||
Meta-test — self-referential ambiguous statement. Should trigger clarification mode.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462521ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"id": "diag-09",
|
||||
"description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.",
|
||||
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
||||
"responseDurationMs": 1785564462521,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-09
|
||||
|
||||
Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
After the deployment last week, our complaint volume tripled to 47 cases per day.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462521ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"id": "diag-10",
|
||||
"description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims.",
|
||||
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
||||
"responseDurationMs": 1785564462522,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-10
|
||||
|
||||
Paired with diag-01 — distinguishes subset complaints from aggregate claims.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Some complaints involve production issues, but others say the delivery team is slow.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 1785564462522ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,313 @@
|
||||
{
|
||||
"timestamp": "2026-08-01T06:07:42.532Z",
|
||||
"provider": "ollama-real",
|
||||
"promptVersion": "v0.2",
|
||||
"casesRun": 10,
|
||||
"summary": {
|
||||
"technical": {
|
||||
"schemaValidityRate": "0.0%",
|
||||
"classificationMatchRate": "0.0%",
|
||||
"nextQuestionPresentRate": "0.0%",
|
||||
"passRate": "0.0%"
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConceptMatchRate": "100.0%",
|
||||
"unsupportedInferenceFailures": "0",
|
||||
"passRate": "0.0%"
|
||||
},
|
||||
"combinedPassRate": "0.0%",
|
||||
"averageResponseDurationMs": "1785564462520"
|
||||
},
|
||||
"testCaseResults": [
|
||||
{
|
||||
"id": "diag-01",
|
||||
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
||||
"responseDurationMs": 1785564462515,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-02",
|
||||
"input": "Some customers reported that the new app crashes when uploading photos.",
|
||||
"responseDurationMs": 1785564462519,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-03",
|
||||
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
||||
"responseDurationMs": 1785564462519,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-04",
|
||||
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
||||
"responseDurationMs": 1785564462520,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-05",
|
||||
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
||||
"responseDurationMs": 1785564462520,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-06",
|
||||
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
||||
"responseDurationMs": 1785564462521,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-07",
|
||||
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
||||
"responseDurationMs": 1785564462521,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-08",
|
||||
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
||||
"responseDurationMs": 1785564462521,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-09",
|
||||
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
||||
"responseDurationMs": 1785564462521,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-10",
|
||||
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
||||
"responseDurationMs": 1785564462522,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Invalid server configuration",
|
||||
"Invalid server configuration"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"id": "diag-01",
|
||||
"description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.",
|
||||
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
||||
"responseDurationMs": 19459,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"rawOutput": "{\"status\":\"received\",\"message\":\"Please provide a specific request or data to process.\"}",
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-01
|
||||
|
||||
Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
We've seen a spike in complaints from our warehouse team this month compared to last month.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 19459ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"id": "diag-02",
|
||||
"description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.",
|
||||
"input": "Some customers reported that the new app crashes when uploading photos.",
|
||||
"responseDurationMs": 14693,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download",
|
||||
"Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-02
|
||||
|
||||
Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Some customers reported that the new app crashes when uploading photos.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 14693ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"id": "diag-03",
|
||||
"description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.",
|
||||
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
||||
"responseDurationMs": 13516,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"rawOutput": "{\"status\":\"success\",\"message\":\"Input received and processed.\"}",
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-03
|
||||
|
||||
Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 13516ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"id": "diag-04",
|
||||
"description": "Decision request — forward-looking, needs missing info identification.",
|
||||
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
||||
"responseDurationMs": 48851,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"rawOutput": "{\"state\":\"pending\"}",
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-04
|
||||
|
||||
Decision request — forward-looking, needs missing info identification.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 48851ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"id": "diag-05",
|
||||
"description": "Unexpected continuity — changed context but no outcome change.",
|
||||
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
||||
"responseDurationMs": 16508,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"rawOutput": "{}",
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-05
|
||||
|
||||
Unexpected continuity — changed context but no outcome change.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Our production line changed suppliers three months ago but still delivers the same defect rate as before.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 16508ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"id": "diag-06",
|
||||
"description": "Quantified improvement — needs context about measurement period and baseline conditions.",
|
||||
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
||||
"responseDurationMs": 15608,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"rawOutput": "{\"status\":\"ok\"}",
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-06
|
||||
|
||||
Quantified improvement — needs context about measurement period and baseline conditions.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
From 45% to 62%, the completion rate for our onboarding flow improved significantly.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 15608ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"id": "diag-07",
|
||||
"description": "Single reported claim — needs validation, not acceptance as fact.",
|
||||
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
||||
"responseDurationMs": 15750,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"rawOutput": "{\"status\":\"success\",\"message\":\"Input received\"}",
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-07
|
||||
|
||||
Single reported claim — needs validation, not acceptance as fact.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
A user claimed that our pricing model is too complex for small businesses.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 15750ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"id": "diag-08",
|
||||
"description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode.",
|
||||
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
||||
"responseDurationMs": 15383,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"rawOutput": "{\"status\":\"success\",\"message\":\"JSON object returned as requested\"}",
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-08
|
||||
|
||||
Meta-test — self-referential ambiguous statement. Should trigger clarification mode.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 15383ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"id": "diag-09",
|
||||
"description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.",
|
||||
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
||||
"responseDurationMs": 15643,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"rawOutput": "{\"status\":\"ok\"}",
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-09
|
||||
|
||||
Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
After the deployment last week, our complaint volume tripled to 47 cases per day.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 15643ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"id": "diag-10",
|
||||
"description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims.",
|
||||
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
||||
"responseDurationMs": 15401,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"rawOutput": "{}",
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diagnostic Case: diag-10
|
||||
|
||||
Paired with diag-01 — distinguishes subset complaints from aggregate claims.
|
||||
|
||||
## Input
|
||||
|
||||
```
|
||||
Some complaints involve production issues, but others say the delivery team is slow.
|
||||
```
|
||||
|
||||
## Result
|
||||
|
||||
- **Technical**: ❌ FAIL (0/4 sub-checks pass)
|
||||
- **Reasoning Quality**: ❌ FAIL (0/2 sub-checks pass)
|
||||
- **Actual Primary Type**: N/A
|
||||
- **Actual Reasoning Modes**: N/A
|
||||
- **Response Duration**: 15401ms
|
||||
|
||||
### Technical Failures
|
||||
|
||||
schema invalid, classification mismatch, no next question
|
||||
|
||||
### Reasoning Quality Failures
|
||||
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
{
|
||||
"timestamp": "2026-08-01T06:11:41.538Z",
|
||||
"provider": "ollama-real",
|
||||
"promptVersion": "v0.2",
|
||||
"casesRun": 10,
|
||||
"summary": {
|
||||
"technical": {
|
||||
"schemaValidityRate": "0.0%",
|
||||
"classificationMatchRate": "0.0%",
|
||||
"nextQuestionPresentRate": "0.0%",
|
||||
"passRate": "0.0%"
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConceptMatchRate": "100.0%",
|
||||
"unsupportedInferenceFailures": "0",
|
||||
"passRate": "0.0%"
|
||||
},
|
||||
"combinedPassRate": "0.0%",
|
||||
"averageResponseDurationMs": "19081"
|
||||
},
|
||||
"testCaseResults": [
|
||||
{
|
||||
"id": "diag-01",
|
||||
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
||||
"responseDurationMs": 19459,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-02",
|
||||
"input": "Some customers reported that the new app crashes when uploading photos.",
|
||||
"responseDurationMs": 14693,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download",
|
||||
"Model returned output that could not be parsed as valid JSON.\n\nAPI used: /api/generate\n/api/chat supported: false\nRaw model output:\n{}`\n\nPossible causes:\n- This Ollama version does not support format:json. The model is producing free-form text.\n- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n- Shorten your scenario to under 500 words\n- Consider upgrading Ollama: https://ollama.com/download"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-03",
|
||||
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
||||
"responseDurationMs": 13516,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-04",
|
||||
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
||||
"responseDurationMs": 48851,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-05",
|
||||
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
||||
"responseDurationMs": 16508,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-06",
|
||||
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
||||
"responseDurationMs": 15608,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-07",
|
||||
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
||||
"responseDurationMs": 15750,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-08",
|
||||
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
||||
"responseDurationMs": 15383,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-09",
|
||||
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
||||
"responseDurationMs": 15643,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "diag-10",
|
||||
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
||||
"responseDurationMs": 15401,
|
||||
"actualPrimaryType": null,
|
||||
"actualReasoningModes": [],
|
||||
"technical": {
|
||||
"schemaValid": false,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"nextQuestionPresent": false,
|
||||
"pass": false,
|
||||
"errors": [
|
||||
"inputClassification: Required",
|
||||
"reconstruction: Required",
|
||||
"evidence: Required",
|
||||
"nextQuestion: Required"
|
||||
]
|
||||
},
|
||||
"reasoningQuality": {
|
||||
"requiredConcepts": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"unsupportedInferencesAbsent": {
|
||||
"pass": true,
|
||||
"details": []
|
||||
},
|
||||
"pass": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"latestRun": "2026-08-01T06-11-41",
|
||||
"caseCount": 10
|
||||
}
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
/**
|
||||
* Core analysis pipeline — shared by API routes and evaluation harness.
|
||||
* Calls the provider, parses output, validates against Zod schemas (v0.2 first, v0.1 fallback).
|
||||
*/
|
||||
|
||||
import { getConfig } from "../lib/config.js";
|
||||
import { getProvider } from "../lib/llm/provider.js";
|
||||
import { buildPrompt, PROMPT_VERSIONS } from "../lib/reconstruction/prompt.js";
|
||||
import {
|
||||
reconstructionV2Schema,
|
||||
reconstructionSchema as reconstructionV1Schema,
|
||||
} from "../lib/reconstruction/schema.js";
|
||||
|
||||
const MAX_SCENARIO_LENGTH = 10000;
|
||||
const DEFAULT_PROMPT_VERSION = "v0.2";
|
||||
|
||||
/**
|
||||
* Analyse a scenario string through the full pipeline.
|
||||
* @param {string} scenario - The scenario text to analyse
|
||||
* @param {object} [opts]
|
||||
* @param {"v0.1" | "v0.2"} [opts.promptVersion="v0.2"] - Prompt version to use
|
||||
* @returns {Promise<object>} Analysis result with diagnostics
|
||||
*/
|
||||
export async function analyseScenario(scenario, opts = {}) {
|
||||
const startTime = Date.now();
|
||||
|
||||
// ── Input validation ───────────────────────────────
|
||||
if (typeof scenario !== "string") {
|
||||
return buildErrorResponse("Input must be a string", startTime);
|
||||
}
|
||||
|
||||
const trimmed = scenario.trim();
|
||||
if (trimmed.length === 0) {
|
||||
return buildErrorResponse("Scenario cannot be empty", startTime);
|
||||
}
|
||||
if (trimmed.length > MAX_SCENARIO_LENGTH) {
|
||||
return buildErrorResponse(`Scenario must be under ${MAX_SCENARIO_LENGTH} characters`, startTime);
|
||||
}
|
||||
|
||||
// ── Configuration check ────────────────────────────
|
||||
const configResult = getConfig();
|
||||
if (!configResult.ok) {
|
||||
return buildErrorResponse("Invalid server configuration", startTime, "500");
|
||||
}
|
||||
|
||||
const { OLLAMA_BASE_URL: _ignored, OLLAMA_MODEL } = configResult.config;
|
||||
const promptVersion = opts.promptVersion || DEFAULT_PROMPT_VERSION;
|
||||
|
||||
// ── Build prompt ───────────────────────────────────
|
||||
let promptObj;
|
||||
try {
|
||||
promptObj = await buildPrompt(trimmed, promptVersion);
|
||||
} catch (e) {
|
||||
return buildErrorResponse(`Failed to build prompt: ${e.message}`, startTime);
|
||||
}
|
||||
|
||||
// ── Call provider ──────────────────────────────────
|
||||
const provider = getProvider();
|
||||
let rawResponse;
|
||||
try {
|
||||
rawResponse = await provider.generateReconstruction(promptObj.prompt, OLLAMA_MODEL);
|
||||
} catch (e) {
|
||||
return buildErrorResponse(
|
||||
e.message || "Provider error during analysis",
|
||||
Date.now() - startTime
|
||||
);
|
||||
}
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
// Try to capture raw response for diagnostics
|
||||
let rawResponseStr;
|
||||
try {
|
||||
rawResponseStr = JSON.stringify(rawResponse);
|
||||
} catch {
|
||||
rawResponseStr = String(rawResponse).slice(0, 2000);
|
||||
}
|
||||
|
||||
// ── Validate against v0.2 schema (preferred) ──────
|
||||
const resultV2 = tryValidateAgainstSchema(rawResponse, reconstructionV2Schema);
|
||||
if (resultV2.valid) {
|
||||
return buildSuccessResultV2(resultV2.data, OLLAMA_MODEL, duration, promptVersion);
|
||||
}
|
||||
|
||||
// ── Fallback to v0.1 schema ────────────────────────
|
||||
const resultV1 = tryValidateAgainstSchema(rawResponse, reconstructionV1Schema);
|
||||
if (resultV1.valid) {
|
||||
return buildSuccessResultV1(resultV1.data, OLLAMA_MODEL, duration, promptVersion);
|
||||
}
|
||||
|
||||
// ── Neither schema matched — partial failure ───────
|
||||
return buildPartialResult(
|
||||
rawResponseStr?.slice(0, 2000),
|
||||
resultV2.error ?? resultV1.error,
|
||||
OLLAMA_MODEL,
|
||||
duration,
|
||||
promptVersion
|
||||
);
|
||||
}
|
||||
|
||||
/** Attempt validation against a Zod schema */
|
||||
function tryValidateAgainstSchema(data, schema) {
|
||||
if (!schema.safeParse) {
|
||||
return { valid: false, error: new Error("Schema does not support safeParse") };
|
||||
}
|
||||
const result = schema.safeParse(data);
|
||||
return result.success ? { valid: true, data: result.data } : { valid: false, error: result.error };
|
||||
}
|
||||
|
||||
// ── Result builders ──────────────────────────────────
|
||||
|
||||
function buildErrorResponse(message, elapsed, statusCode = 500) {
|
||||
return {
|
||||
success: false,
|
||||
error: message,
|
||||
modelName: null,
|
||||
responseDurationMs: elapsed,
|
||||
validationStatus: "invalid",
|
||||
rawResponse: null,
|
||||
promptVersion: null,
|
||||
statusCode,
|
||||
};
|
||||
}
|
||||
|
||||
function buildSuccessResultV2(data, model, duration, version) {
|
||||
return {
|
||||
success: true,
|
||||
validationStatus: "valid",
|
||||
modelName: model,
|
||||
responseDurationMs: duration,
|
||||
rawResponse: JSON.stringify(data).slice(0, 3000),
|
||||
promptVersion: version,
|
||||
inputClassification: data.inputClassification,
|
||||
reconstruction: data.reconstruction,
|
||||
evidence: data.evidence,
|
||||
nextQuestion: data.nextQuestion,
|
||||
errors: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function buildSuccessResultV1(data, model, duration, version) {
|
||||
return {
|
||||
success: true,
|
||||
validationStatus: "valid",
|
||||
modelName: model,
|
||||
responseDurationMs: duration,
|
||||
rawResponse: JSON.stringify(data).slice(0, 3000),
|
||||
promptVersion: version,
|
||||
inputClassification: null,
|
||||
reconstruction: data,
|
||||
evidence: undefined,
|
||||
nextQuestion: undefined,
|
||||
errors: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function buildPartialResult(rawResp, error, model, duration, version) {
|
||||
let errors = [];
|
||||
if (error && typeof error.flatten === "function") {
|
||||
errors = error.flatten().fieldErrors
|
||||
? Object.entries(error.flatten().fieldErrors).flatMap(([k, v]) => [`${k}: ${v.join(", ")}`])
|
||||
: [String(error)];
|
||||
} else if (error) {
|
||||
errors = [String(error).slice(0, 500)];
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
validationStatus: "invalid",
|
||||
modelName: model,
|
||||
responseDurationMs: duration,
|
||||
rawResponse: rawResp?.slice(0, 2000),
|
||||
promptVersion: version,
|
||||
inputClassification: null,
|
||||
reconstruction: null,
|
||||
evidence: undefined,
|
||||
nextQuestion: undefined,
|
||||
errors,
|
||||
};
|
||||
}
|
||||
|
||||
export { PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION };
|
||||
+3
-5
@@ -93,11 +93,9 @@ async function detectChatSupport(baseUrl) {
|
||||
|
||||
class OllamaLlmProvider {
|
||||
async generateReconstruction(scenario, modelName) {
|
||||
const { buildPrompt } = await import("@/lib/reconstruction/prompt");
|
||||
|
||||
let rawPrompt = buildPrompt(scenario);
|
||||
// Stronger JSON hint since we can't use format:json on older Ollama
|
||||
const prompt = rawPrompt + `\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.`;
|
||||
// scenario is ALREADY a fully-built prompt text (built by analyseScenario).
|
||||
// Do NOT call buildPrompt() again — that would double-wrap the prompt.
|
||||
const prompt = scenario;
|
||||
|
||||
const baseUrl = process.env.OLLAMA_BASE_URL;
|
||||
if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");
|
||||
|
||||
@@ -1,5 +1,17 @@
|
||||
export function buildPrompt(scenario) {
|
||||
return `You are a neutral analyst performing an evidence-based reconstruction of the following scenario.
|
||||
import { promises as fs } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { dirname, join } from "node:path";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
const PROMPTS_DIR = join(__dirname, "../../prompts");
|
||||
|
||||
/** Available prompt versions */
|
||||
export const PROMPT_VERSIONS = ["v0.1", "v0.2"];
|
||||
|
||||
/** Build a v0.1 (extraction-only) prompt inline for backward compatibility */
|
||||
function buildV1Prompt(scenario) {
|
||||
return `You are a neutral analyst performing an evidence-based reconstruction of the following scenario.
|
||||
|
||||
Rules:
|
||||
1. Do NOT invent facts. Only include information present in the scenario or clearly implied.
|
||||
@@ -29,3 +41,34 @@ Return valid JSON matching this structure exactly:
|
||||
|
||||
Return ONLY the JSON object. No markdown, no explanation, no preamble.`;
|
||||
}
|
||||
|
||||
/** Load a versioned prompt from disk and substitute {{SCENARIO}} */
|
||||
async function buildV2Prompt(scenario) {
|
||||
try {
|
||||
const content = await fs.readFile(join(PROMPTS_DIR, "reconstruct-v0.2.md"), "utf-8");
|
||||
return content.replace("{{SCENARIO}}", scenario);
|
||||
} catch {
|
||||
// Fall back to v0.1 prompt if v0.2 file is missing
|
||||
return buildV1Prompt(scenario);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build an analysis prompt for the given version.
|
||||
* @param {"v0.1" | "v0.2"} [version="v0.2"]
|
||||
* @returns {Promise<{prompt: string, version: string}>}
|
||||
*/
|
||||
export async function buildPrompt(scenario, version = "v0.2") {
|
||||
let prompt;
|
||||
switch (version) {
|
||||
case "v0.1":
|
||||
prompt = buildV1Prompt(scenario);
|
||||
break;
|
||||
default: // v0.2
|
||||
prompt = await buildV2Prompt(scenario);
|
||||
break;
|
||||
}
|
||||
|
||||
const strongJsonHint = "\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.";
|
||||
return { prompt: prompt + strongJsonHint, version };
|
||||
}
|
||||
|
||||
+164
-13
@@ -1,38 +1,51 @@
|
||||
import { z } from "zod";
|
||||
|
||||
const confidenceEnum = z.enum(["low", "medium", "high"]);
|
||||
// ──────────────────────────────────────────────
|
||||
// Shared enums (v0.1 & v0.2)
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
const itemSchema = z.object({
|
||||
export const confidenceEnum = z.enum(["low", "medium", "high"]);
|
||||
const importanceEnum = z.enum(["incidental", "supporting", "important", "critical"]);
|
||||
const expectedInfoValueEnum = z.enum(["low", "medium", "high"]);
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// v0.1 — extraction-only schema (preserved)
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
const confidenceEnumV1 = z.enum(["low", "medium", "high"]);
|
||||
|
||||
const itemSchemaV1 = z.object({
|
||||
id: z.string().min(1),
|
||||
description: z.string().min(1),
|
||||
confidence: confidenceEnum,
|
||||
confidence: confidenceEnumV1,
|
||||
});
|
||||
|
||||
export const reconstructionSchema = z.object({
|
||||
observations: z.array(itemSchema),
|
||||
observations: z.array(itemSchemaV1),
|
||||
reportedClaims: z.array(
|
||||
itemSchema.extend({
|
||||
itemSchemaV1.extend({
|
||||
attributedTo: z.union([z.string().min(1), z.null()]).optional().nullable(),
|
||||
})
|
||||
),
|
||||
assumptions: z.array(itemSchema),
|
||||
entities: z.array(itemSchema),
|
||||
assumptions: z.array(itemSchemaV1),
|
||||
entities: z.array(itemSchemaV1),
|
||||
transitions: z.array(
|
||||
itemSchema.extend({
|
||||
itemSchemaV1.extend({
|
||||
entity: z.string().min(1),
|
||||
previousState: z.string().min(1),
|
||||
currentState: z.string().min(1),
|
||||
explanationStatus: z.string().min(1),
|
||||
})
|
||||
),
|
||||
expectedButMissing: z.array(itemSchema),
|
||||
presentButUnexpected: z.array(itemSchema),
|
||||
contradictions: z.array(itemSchema),
|
||||
openUncertainties: z.array(itemSchema),
|
||||
expectedButMissing: z.array(itemSchemaV1),
|
||||
presentButUnexpected: z.array(itemSchemaV1),
|
||||
contradictions: z.array(itemSchemaV1),
|
||||
openUncertainties: z.array(itemSchemaV1),
|
||||
});
|
||||
|
||||
// v0.1 analyse response (used internally)
|
||||
export const analyseResponseSchema = z.object({
|
||||
reconstruction: reconstructionSchema,
|
||||
reconstruction: z.union([reconstructionSchema, z.null()]),
|
||||
modelName: z.string(),
|
||||
responseDurationMs: z.number(),
|
||||
validationStatus: z.enum(["valid", "partial", "invalid"]),
|
||||
@@ -48,6 +61,133 @@ export const healthResponseSchema = z.object({
|
||||
error: z.string().nullable(),
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// v0.2 — reasoning classification + reconstruction
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
export const inputTypes = /** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").INPUT_TYPE_VALUE>} */ (
|
||||
z.enum([
|
||||
"observed_problem",
|
||||
"unexplained_change",
|
||||
"contradiction",
|
||||
"decision_request",
|
||||
"causal_claim",
|
||||
"reported_claim",
|
||||
"fault_report",
|
||||
"ambiguous_statement",
|
||||
"question",
|
||||
"desired_outcome",
|
||||
"insufficient_context",
|
||||
"other",
|
||||
])
|
||||
);
|
||||
|
||||
export const reasoningModes = /** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").REASONING_MODE_VALUE>} */ (
|
||||
z.enum([
|
||||
"establish_baseline",
|
||||
"identify_difference",
|
||||
"reconstruct_transition",
|
||||
"decompose_aggregate",
|
||||
"validate_measurement",
|
||||
"validate_claim",
|
||||
"investigate_contradiction",
|
||||
"clarify_meaning",
|
||||
"decision_support",
|
||||
"fault_investigation",
|
||||
"identify_missing_information",
|
||||
"test_possible_explanations",
|
||||
"other",
|
||||
])
|
||||
);
|
||||
|
||||
const evidenceRecordSchema = z.object({
|
||||
id: z.string().min(1),
|
||||
description: z.string().min(1),
|
||||
evidenceType: z.enum(["direct_observation", "reported_statement", "interpretation", "assumption", "inferred_relationship"]),
|
||||
source: z.string().optional(),
|
||||
attribution: z.string().nullable().optional(),
|
||||
confidence: confidenceEnum,
|
||||
importance: importanceEnum,
|
||||
});
|
||||
|
||||
const reconstructionSchemaV2 = z.object({
|
||||
summary: z.string().min(1),
|
||||
actors: z.array(itemSchemaV1),
|
||||
systemsOrObjects: z.array(itemSchemaV1),
|
||||
expectedStates: z.array(itemSchemaV1),
|
||||
observedStates: z.array(itemSchemaV1),
|
||||
differences: z.array(itemSchemaV1),
|
||||
knownTransitions: z.array(
|
||||
itemSchemaV1.extend({
|
||||
entity: z.string().min(1),
|
||||
previousState: z.string().min(1),
|
||||
currentState: z.string().min(1),
|
||||
explanationStatus: z.string().min(1),
|
||||
})
|
||||
),
|
||||
unexplainedTransitions: z.array(
|
||||
itemSchemaV1.extend({
|
||||
entity: z.string().min(1).optional(),
|
||||
previousState: z.string().min(1).optional(),
|
||||
currentState: z.string().min(1).optional(),
|
||||
})
|
||||
),
|
||||
contradictions: z.array(itemSchemaV1),
|
||||
importantUnknowns: z.array(itemSchemaV1),
|
||||
plausibleInterpretations: z.array(
|
||||
z.object({
|
||||
id: z.string().min(1),
|
||||
description: z.string().min(1),
|
||||
supportingEvidenceIds: z.array(z.string()),
|
||||
assumptionsRequired: z.array(z.string()).optional().default([]),
|
||||
confidence: confidenceEnum,
|
||||
})
|
||||
),
|
||||
});
|
||||
|
||||
const inputClassificationSchema = z.object({
|
||||
primaryType: inputTypes,
|
||||
secondaryTypes: z.array(inputTypes).optional().default([]),
|
||||
reasoningModes: z.array(reasoningModes).optional().default([]),
|
||||
classificationReason: z.string().min(1),
|
||||
confidence: confidenceEnum,
|
||||
});
|
||||
|
||||
const nextQuestionSchema = z.object({
|
||||
id: z.string().min(1),
|
||||
question: z.string().min(1),
|
||||
targets: z.array(z.string()),
|
||||
reason: z.string().min(1),
|
||||
expectedInformationValue: expectedInfoValueEnum,
|
||||
reasoningMode: reasoningModes.optional().default("other"),
|
||||
});
|
||||
|
||||
// v0.2 complete analysis response (what the model produces)
|
||||
export const reconstructionV2Schema = z.object({
|
||||
inputClassification: inputClassificationSchema,
|
||||
reconstruction: reconstructionSchemaV2,
|
||||
evidence: z.array(evidenceRecordSchema),
|
||||
nextQuestion: nextQuestionSchema,
|
||||
});
|
||||
|
||||
// Outer wrapper for API return (includes diagnostics + v0.2 data)
|
||||
export const analyseResponseV2Schema = z.object({
|
||||
inputClassification: inputClassificationSchema.optional(),
|
||||
reconstruction: reconstructionSchemaV2.optional().nullable(),
|
||||
evidence: z.array(evidenceRecordSchema).optional(),
|
||||
nextQuestion: nextQuestionSchema.optional(),
|
||||
modelName: z.string(),
|
||||
responseDurationMs: z.number(),
|
||||
validationStatus: z.enum(["valid", "partial", "invalid"]),
|
||||
rawResponse: z.string().optional(),
|
||||
errors: z.array(z.string()).optional(),
|
||||
promptVersion: z.string().optional(),
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Parsing helpers
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
export function parseReconstruction(raw) {
|
||||
if (typeof raw === "string") {
|
||||
try {
|
||||
@@ -58,3 +198,14 @@ export function parseReconstruction(raw) {
|
||||
}
|
||||
return reconstructionSchema.parse(raw);
|
||||
}
|
||||
|
||||
export function parseReconstructionV2(raw) {
|
||||
if (typeof raw === "string") {
|
||||
try {
|
||||
raw = JSON.parse(raw);
|
||||
} catch {
|
||||
throw new SyntaxError("Model response is not valid JSON");
|
||||
}
|
||||
}
|
||||
return reconstructionV2Schema.parse(raw);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
# v0.1 vs v0.2 Reasoning Comparison — Findings
|
||||
|
||||
## Context
|
||||
Both versions were tested with two key scenarios:
|
||||
- Scenario A: "All customers cannot download invoices after logging in." (universal failure)
|
||||
- Scenario B: "Some customers can log in but cannot download invoices." (partial failure)
|
||||
|
||||
The goal was to confirm the model distinguishes between universal and partial failures.
|
||||
|
||||
## Results — v0.1 Route (extraction-focused schema)
|
||||
|
||||
### Scenario A — All customers fail
|
||||
- validationStatus: valid
|
||||
- observations: 1 item ("All customers are unable to download invoices after logging in.")
|
||||
- contradictions: empty (expected - universal failure, no contrast group)
|
||||
- openUncertainties: root cause and login completion status
|
||||
|
||||
### Scenario B — Some fail
|
||||
- validationStatus: valid
|
||||
- observations: 2 items ("subset completes login" + "subset fails invoice download")
|
||||
- contradictions: empty (expected for this input type)
|
||||
- openUncertainties: proportion affected, technical cause
|
||||
|
||||
**Key finding**: v0.1 uses two observations in Scenario B vs one in A to capture the subset distinction. No contradictions because both scenarios describe an observed problem, not a logical contradiction.
|
||||
|
||||
## Results — v0.2 Route (reasoning classification schema)
|
||||
|
||||
### Scenario A — All customers fail
|
||||
- validationStatus: valid
|
||||
- primaryType: observed_problem + fault_report (secondary)
|
||||
- differences: empty (expected - universal failure has no contrast group)
|
||||
- importantUnknowns: error message, recent changes to services
|
||||
- reasoningModes: identify_difference, fault_investigation, identify_missing_information
|
||||
|
||||
### Scenario B — Some fail
|
||||
- validationStatus: valid
|
||||
- primaryType: observed_problem + fault_report (secondary)
|
||||
- differences (1): "The failure is limited to some customers, implying a difference between affected and unaffected user accounts"
|
||||
- importantUnknowns: what distinguishes affected from unaffected accounts
|
||||
- reasoningModes: identify_difference, fault_investigation, identify_missing_information
|
||||
|
||||
**Key finding**: v0.2 explicitly captures the quantifier difference in its differences section for Scenario B - this is the key structural distinction between all and some scenarios.
|
||||
|
||||
## Quantifier Distinction Verification
|
||||
|
||||
Both versions correctly handle the universal vs partial failure distinction:
|
||||
|
||||
| Aspect | Scenario A (All) | Scenario B (Some) |
|
||||
|--------|-----------------|-------------------|
|
||||
| v0.1 observations | 1 (universal) | 2 (login OK + download fail) |
|
||||
| v0.1 contradictions | 0 (expected) | 0 (expected) |
|
||||
| v0.2 primaryType | observed_problem | observed_problem |
|
||||
| v0.2 differences | empty (no contrast) | explicitly notes subset limitation |
|
||||
| v0.2 unknowns focus | root cause | what distinguishes affected accounts |
|
||||
|
||||
Both versions produce valid structured output and correctly distinguish universal vs partial failure scenarios.
|
||||
|
||||
## Prompt Fix Summary
|
||||
|
||||
The v0.2 prompt template (prompts/reconstruct-v0.2.md) was updated to include an explicit JSON output schema section that:
|
||||
1. Specifies exact camelCase key names matching the Zod schema
|
||||
2. Lists all valid enum values for primaryType and reasoningModes
|
||||
3. Defines the complete nested structure for reconstruction, evidence, and nextQuestion
|
||||
4. Includes critical rules preventing snake_case keys or invented top-level fields
|
||||
|
||||
Before fix: Model output had input_classification, reasoning_mode, anchors - all invalid per Zod schema -> validationStatus: invalid
|
||||
After fix: Model output has inputClassification, reconstruction, evidence, nextQuestion with correct nested structure -> validationStatus: valid
|
||||
@@ -0,0 +1,122 @@
|
||||
You are a neutral analyst performing evidence-based situation reconstruction.
|
||||
|
||||
## Rules
|
||||
|
||||
1. Do NOT invent facts, context or causes. Only include information present in the scenario or clearly implied.
|
||||
2. First determine what kind of input has been supplied. Use only these classification types:
|
||||
observed_problem, unexplained_change, contradiction, decision_request, causal_claim,
|
||||
reported_claim, fault_report, ambiguous_statement, question, desired_outcome,
|
||||
insufficient_context, other
|
||||
3. Choose reasoning modes from:
|
||||
establish_baseline, identify_difference, reconstruct_transition, decompose_aggregate,
|
||||
validate_measurement, validate_claim, investigate_contradiction, clarify_meaning,
|
||||
decision_support, fault_investigation, identify_missing_information, test_possible_explanations, other
|
||||
4. Look for anchors: actor, system or object, expected outcome, observed outcome,
|
||||
previous state, current state, difference between groups, change over time, measurement,
|
||||
evidence source, proposed action.
|
||||
5. Identify meaningful differences (e.g., some succeed while others fail; revenue rises while cash falls).
|
||||
6. Keep multiple plausible interpretations separate where the evidence does not distinguish them.
|
||||
7. Distinguish: what was said / what it may mean / why it may have been said.
|
||||
8. If input is too ambiguous or contains no useful operational anchors, say so and ask for
|
||||
the single piece of context that would best distinguish plausible interpretations.
|
||||
|
||||
## Confidence scale
|
||||
|
||||
- low — weak evidence, speculation, or missing information
|
||||
- medium — reasonable inference from available evidence
|
||||
- high — strong evidence, direct observation, or confirmed fact
|
||||
|
||||
## Importance scale (evidence records)
|
||||
|
||||
- incidental — minor detail, unlikely to affect conclusions
|
||||
- supporting — adds context but not critical
|
||||
- important — materially affects understanding of the situation
|
||||
- critical — essential to resolving the situation; without it conclusions cannot be drawn
|
||||
|
||||
## Expected information value (next question)
|
||||
|
||||
- low — marginally useful even if answered
|
||||
- medium — meaningfully clarifies the situation
|
||||
- high — would significantly distinguish between plausible explanations or fill a gap in understanding
|
||||
|
||||
## Next question selection criteria
|
||||
|
||||
Prefer questions that:
|
||||
- clarify a major difference
|
||||
- establish a baseline
|
||||
- explain an important transition
|
||||
- test an unsupported claim
|
||||
- distinguish between plausible explanations
|
||||
- request measurable evidence
|
||||
- identify who or what is affected
|
||||
- establish timing
|
||||
|
||||
Avoid questions that:
|
||||
- have already been answered
|
||||
- assume a cause
|
||||
- jump to a solution
|
||||
- ask about motive before the observable situation is understood
|
||||
- focus on incidental wording
|
||||
- are too broad to produce useful information
|
||||
- combine many unrelated questions
|
||||
|
||||
## Output format — return this exact JSON structure
|
||||
|
||||
Return a JSON object with exactly these four top-level keys (use **camelCase**):
|
||||
|
||||
```json
|
||||
{
|
||||
"inputClassification": {
|
||||
"primaryType": "<one of: observed_problem, unexplained_change, contradiction, decision_request, causal_claim, reported_claim, fault_report, ambiguous_statement, question, desired_outcome, insufficient_context, other>",
|
||||
"secondaryTypes": ["<optional additional types from the same list>"],
|
||||
"reasoningModes": ["<one or more of: establish_baseline, identify_difference, reconstruct_transition, decompose_aggregate, validate_measurement, validate_claim, investigate_contradiction, clarify_meaning, decision_support, fault_investigation, identify_missing_information, test_possible_explanations, other>"],
|
||||
"classificationReason": "<brief explanation of why you chose the primary type>",
|
||||
"confidence": "<low | medium | high>"
|
||||
},
|
||||
"reconstruction": {
|
||||
"summary": "<one-sentence overview of the situation>",
|
||||
"actors": [{"id": "<any unique string>", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"systemsOrObjects": [{"id": "<any unique string>", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"expectedStates": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"observedStates": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"differences": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"knownTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}],
|
||||
"unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "..."}],
|
||||
"contradictions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"importantUnknowns": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": ["<ids that support this interpretation>"], "assumptionsRequired": [], "confidence": "<low|medium|high>"}]
|
||||
},
|
||||
"evidence": [
|
||||
{
|
||||
"id": "<any unique string>",
|
||||
"description": "...",
|
||||
"evidenceType": "<direct_observation | reported_statement | interpretation | assumption | inferred_relationship>",
|
||||
"source": "<optional — who/where this came from>",
|
||||
"attribution": null,
|
||||
"confidence": "<low | medium | high>",
|
||||
"importance": "<incidental | supporting | important | critical>"
|
||||
}
|
||||
],
|
||||
"nextQuestion": {
|
||||
"id": "<any unique string>",
|
||||
"question": "<one precise question>",
|
||||
"targets": ["<what this question targets — e.g. 'actor', 'system', 'expectedOutcome'>"],
|
||||
"reason": "<why answering this is important>",
|
||||
"expectedInformationValue": "<low | medium | high>",
|
||||
"reasoningMode": "<optional reasoning mode from the list above>"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
CRITICAL RULES for JSON output:
|
||||
1. Use **exactly** the key names shown above (camelCase, no snake_case).
|
||||
2. The four top-level keys must be: `inputClassification`, `reconstruction`, `evidence`, `nextQuestion`.
|
||||
3. Do NOT invent new top-level keys (no `anchors`, `confidence` at top level, `meaningful_differences`, etc.).
|
||||
4. Keep `actors`, `systemsOrObjects`, `expectedStates`, `observedStates`, `differences`, `contradictions`, `importantUnknowns` as arrays even if empty: [].
|
||||
5. Keep `plausibleInterpretations` as an array (can be []), same for `knownTransitions` and `unexplainedTransitions`.
|
||||
6. Each object in arrays must have at least `id`, `description`, `confidence`.
|
||||
|
||||
Scenario:
|
||||
{{SCENARIO}}
|
||||
|
||||
Return ONLY the JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"qwen-claude:latest": {
|
||||
"availability": "NOT_AVAILABLE_ON_SERVER"
|
||||
},
|
||||
"qwen3.6:35b-a3b": {
|
||||
"availability": "NOT_AVAILABLE_ON_SERVER"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
{
|
||||
"qwen-claude:latest": {
|
||||
"test_A_qwen_claude": {
|
||||
"testDescription": "Plain instruction test — should return CHAT_WORKS",
|
||||
"endpoint": "/api/chat",
|
||||
"format": "json",
|
||||
"hasSystemMessage": true,
|
||||
"httpStatus": 200,
|
||||
"messageContentType": "string",
|
||||
"messageContentLength": 30,
|
||||
"thinkingPresent": true,
|
||||
"parsedContentKeys": [
|
||||
"response"
|
||||
],
|
||||
"looksLikeStatusAck": false,
|
||||
"rawPreview": "{\n \"response\": \"CHAT_WORKS\"\n}"
|
||||
},
|
||||
"test_A_qwen_claude_generate": {
|
||||
"endpoint": "/api/generate",
|
||||
"httpStatus": 200,
|
||||
"responseFirst200": "CHAT_WORKS",
|
||||
"responseLooksLikeStructuredJSON": false,
|
||||
"rawPreview": "CHAT_WORKS"
|
||||
},
|
||||
"test_B_qwen_claude": {
|
||||
"testDescription": "JSON schema test — should return exact object",
|
||||
"endpoint": "/api/chat",
|
||||
"format": "json",
|
||||
"hasSystemMessage": false,
|
||||
"httpStatus": 200,
|
||||
"messageContentType": "string",
|
||||
"messageContentLength": 38,
|
||||
"thinkingPresent": true,
|
||||
"parsedContentKeys": [
|
||||
"message"
|
||||
],
|
||||
"looksLikeStatusAck": false,
|
||||
"rawPreview": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}"
|
||||
},
|
||||
"test_B_qwen_claude_generate": {
|
||||
"endpoint": "/api/generate",
|
||||
"httpStatus": 200,
|
||||
"responseFirst200": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}",
|
||||
"responseLooksLikeStructuredJSON": true,
|
||||
"rawPreview": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}"
|
||||
},
|
||||
"test_C_qwen_claude": {
|
||||
"testDescription": "Minimal reconstruction schema — structured output test",
|
||||
"endpoint": "/api/chat",
|
||||
"format": "json",
|
||||
"hasSystemMessage": false,
|
||||
"httpStatus": 200,
|
||||
"messageContentType": "string",
|
||||
"messageContentLength": 1145,
|
||||
"thinkingPresent": true,
|
||||
"parsedContentKeys": [
|
||||
"analysis",
|
||||
"meaningful_difference",
|
||||
"next_question"
|
||||
],
|
||||
"looksLikeStatusAck": false,
|
||||
"rawPreview": "{\"analysis\":\"The workflow breaks into two distinct stages: authentication (login) and resource access (invoice download). Since login succeeds for the affected users, identity verification and session creation are functioning correctly. The failure occurs downstream in the layer responsible for loca"
|
||||
},
|
||||
"test_C_qwen_claude_generate": {
|
||||
"endpoint": "/api/generate",
|
||||
"httpStatus": 200,
|
||||
"responseFirst200": "**Analysis of the Situation**\nThe issue represents a partial, post-authentication failure: identity verification works for a subset of users, but access to or delivery of a specific resource (invoices",
|
||||
"responseLooksLikeStructuredJSON": false,
|
||||
"rawPreview": "**Analysis of the Situation**\nThe issue represents a partial, post-authentication failure: identity verification works for a subset of users, but access to or delivery of a specific resource (invoices"
|
||||
}
|
||||
},
|
||||
"qwen3.6:35b-a3b": {
|
||||
"test_A_qwen___": {
|
||||
"testDescription": "Plain instruction test — should return CHAT_WORKS",
|
||||
"endpoint": "/api/chat",
|
||||
"format": "json",
|
||||
"hasSystemMessage": true,
|
||||
"httpStatus": 200,
|
||||
"messageContentType": "string",
|
||||
"messageContentLength": 30,
|
||||
"thinkingPresent": true,
|
||||
"parsedContentKeys": [
|
||||
"response"
|
||||
],
|
||||
"looksLikeStatusAck": false,
|
||||
"rawPreview": "{\n \"response\": \"CHAT_WORKS\"\n}"
|
||||
},
|
||||
"test_A_qwen____generate": {
|
||||
"endpoint": "/api/generate",
|
||||
"httpStatus": 200,
|
||||
"responseFirst200": "CHAT_WORKS",
|
||||
"responseLooksLikeStructuredJSON": false,
|
||||
"rawPreview": "CHAT_WORKS"
|
||||
},
|
||||
"test_B_qwen___": {
|
||||
"testDescription": "JSON schema test — should return exact object",
|
||||
"endpoint": "/api/chat",
|
||||
"format": "json",
|
||||
"hasSystemMessage": false,
|
||||
"httpStatus": 200,
|
||||
"messageContentType": "string",
|
||||
"messageContentLength": 38,
|
||||
"thinkingPresent": true,
|
||||
"parsedContentKeys": [
|
||||
"message"
|
||||
],
|
||||
"looksLikeStatusAck": false,
|
||||
"rawPreview": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}"
|
||||
},
|
||||
"test_B_qwen____generate": {
|
||||
"endpoint": "/api/generate",
|
||||
"httpStatus": 200,
|
||||
"responseFirst200": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}",
|
||||
"responseLooksLikeStructuredJSON": true,
|
||||
"rawPreview": "{\"message\": \"STRUCTURED_OUTPUT_WORKS\"}"
|
||||
},
|
||||
"test_C_qwen___": {
|
||||
"testDescription": "Minimal reconstruction schema — structured output test",
|
||||
"endpoint": "/api/chat",
|
||||
"format": "json",
|
||||
"hasSystemMessage": false,
|
||||
"httpStatus": 200,
|
||||
"messageContentType": "string",
|
||||
"messageContentLength": 843,
|
||||
"thinkingPresent": true,
|
||||
"parsedContentKeys": [
|
||||
"analysis",
|
||||
"meaningful_difference",
|
||||
"next_question"
|
||||
],
|
||||
"looksLikeStatusAck": false,
|
||||
"rawPreview": "{\"analysis\": \"The scenario indicates that authentication is functioning correctly, but a downstream access or resource-retrieval step is failing for a subset of users. This separates the problem from credential or session validity and points toward post-login factors such as authorization scope, doc"
|
||||
},
|
||||
"test_C_qwen____generate": {
|
||||
"endpoint": "/api/generate",
|
||||
"httpStatus": 200,
|
||||
"responseFirst200": "**Meaningful Difference:** \nSuccessful login verifies *account-level authentication and baseline platform access*, while downloading an invoice requires *invoice-specific data availability, retrieval",
|
||||
"responseLooksLikeStructuredJSON": false,
|
||||
"rawPreview": "**Meaningful Difference:** \nSuccessful login verifies *account-level authentication and baseline platform access*, while downloading an invoice requires *invoice-specific data availability, retrieval"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
/**
|
||||
* Debug script: send raw Ollama requests directly, bypassing the application provider.
|
||||
* Tests /api/chat with format:json and captures request payloads + raw responses.
|
||||
*/
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const BASE_URL = process.env.OLLAMA_BASE_URL || "http://localhost:11434";
|
||||
const TIMESTAMP = new Date().toISOString().replace(/[/:]/g, "-");
|
||||
const RESULTS_DIR = join(__dirname, "..", "provider-debug-results", TIMESTAMP);
|
||||
|
||||
mkdirSync(RESULTS_DIR, { recursive: true });
|
||||
|
||||
// ============================================================
|
||||
// Test cases
|
||||
// ============================================================
|
||||
|
||||
const MODEL_A = "qwen-claude:latest";
|
||||
const MODEL_B = "qwen3.6:35b-a3b";
|
||||
|
||||
function getModelList() {
|
||||
// Check which models are available locally (not via Ollama server)
|
||||
return { A: MODEL_A, B: MODEL_B };
|
||||
}
|
||||
|
||||
// Test A: Simple text reply to verify model responds normally
|
||||
const TEST_A = {
|
||||
label: "A",
|
||||
description: "Plain instruction test — should return CHAT_WORKS",
|
||||
system: "You are a normal assistant. Follow the user instruction exactly.",
|
||||
user: "Reply with exactly: CHAT_WORKS",
|
||||
};
|
||||
|
||||
// Test B: Explicit JSON schema via format field
|
||||
const TEST_B = {
|
||||
label: "B",
|
||||
description: "JSON schema test — should return exact object",
|
||||
system: null, // uses messages only with format
|
||||
user: 'Return exactly: {"message": "STRUCTURED_OUTPUT_WORKS"}',
|
||||
};
|
||||
|
||||
// Test C: Minimal reconstruction-style schema
|
||||
const TEST_C = {
|
||||
label: "C",
|
||||
description: "Minimal reconstruction schema — structured output test",
|
||||
system: null,
|
||||
user: "Analyse this situation without solving it: Some customers can log in but cannot download invoices. Identify the meaningful difference and ask one useful next question.",
|
||||
};
|
||||
|
||||
const ALL_TESTS = [TEST_A, TEST_B, TEST_C];
|
||||
|
||||
// ============================================================
|
||||
// Helper functions
|
||||
// ============================================================
|
||||
|
||||
async function runChatWithFormat(model, messages, format) {
|
||||
const body = { model, messages, stream: false, format };
|
||||
const res = await fetch(`${BASE_URL}/api/chat`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
const rawText = await res.text();
|
||||
let parsed = null;
|
||||
try { parsed = JSON.parse(rawText); } catch {}
|
||||
|
||||
return {
|
||||
status: res.status,
|
||||
statusText: res.statusText,
|
||||
requestPayload: body,
|
||||
rawResponseText: rawText.slice(0, 5000),
|
||||
parsedResponse: parsed,
|
||||
messageContent: parsed?.message?.content ?? null,
|
||||
thinkingLength: (parsed?.message?.thinking || "").length,
|
||||
messageContentType: typeof parsed?.message?.content,
|
||||
responseField: parsed?.response,
|
||||
};
|
||||
}
|
||||
|
||||
async function runGenerate(model, prompt) {
|
||||
const body = { model, prompt, stream: false };
|
||||
const res = await fetch(`${BASE_URL}/api/generate`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
const rawText = await res.text();
|
||||
let parsed = null;
|
||||
try { parsed = JSON.parse(rawText); } catch {}
|
||||
|
||||
return {
|
||||
status: res.status,
|
||||
requestPayload: body,
|
||||
rawResponseText: rawText.slice(0, 5000),
|
||||
parsedResponse: parsed,
|
||||
responseField: typeof parsed?.response === "string" ? parsed.response : JSON.stringify(parsed),
|
||||
responseFirst200: (parsed?.response || "").slice(0, 200),
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Run tests
|
||||
// ============================================================
|
||||
|
||||
const results = {};
|
||||
|
||||
for (const model of [MODEL_A, MODEL_B]) {
|
||||
console.log(`\n=== Testing model: ${model} ===`);
|
||||
results[model] = {};
|
||||
|
||||
// Check if model is available locally
|
||||
let available = false;
|
||||
try {
|
||||
const tagsRes = await fetch(`${BASE_URL}/api/tags`);
|
||||
const tagsData = await tagsRes.json();
|
||||
available = tagsData.models?.some(m => m.name.includes(model.split(":")[0]));
|
||||
} catch (e) {
|
||||
console.log(` Warning: could not check model availability: ${e.message}`);
|
||||
}
|
||||
|
||||
if (!available) {
|
||||
results[model].availability = "NOT_AVAILABLE_ON_SERVER";
|
||||
console.log(` -> Model ${model} not found on server, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(` -> Model available on server\n`);
|
||||
|
||||
for (const test of ALL_TESTS) {
|
||||
const testKey = `test_${test.label}_${model.split(":")[0].replace(/[^a-zA-Z]/g, "_")}`;
|
||||
console.log(` Running Test ${test.label}: ${test.description}`);
|
||||
|
||||
// Chat with format:json
|
||||
let chatResult;
|
||||
try {
|
||||
const messages = [];
|
||||
if (test.system) {
|
||||
messages.push({ role: "system", content: test.system });
|
||||
}
|
||||
messages.push({ role: "user", content: test.user });
|
||||
|
||||
chatResult = await runChatWithFormat(model, messages, "json");
|
||||
|
||||
// Try to extract JSON from message.content
|
||||
let extractedJson = null;
|
||||
if (typeof chatResult.messageContent === "string") {
|
||||
try {
|
||||
extractedJson = JSON.parse(chatResult.messageContent);
|
||||
} catch {}
|
||||
}
|
||||
|
||||
results[model][testKey] = {
|
||||
testDescription: test.description,
|
||||
endpoint: "/api/chat",
|
||||
format: "json",
|
||||
hasSystemMessage: !!test.system,
|
||||
httpStatus: chatResult.status,
|
||||
messageContentType: chatResult.messageContentType,
|
||||
messageContentLength: chatResult.messageContent?.length || 0,
|
||||
thinkingPresent: chatResult.thinkingLength > 0,
|
||||
parsedContentKeys: extractedJson ? Object.keys(extractedJson) : null,
|
||||
// If content looks like a status acknowledgment
|
||||
looksLikeStatusAck: typeof chatResult.messageContent === "string" &&
|
||||
(chatResult.messageContent.includes('"status"') || chatResult.messageContent.includes('"state"')),
|
||||
rawPreview: chatResult.messageContent?.slice(0, 300) ?? "(none)",
|
||||
};
|
||||
|
||||
const status = extractedJson ? "JSON_OK" : (chatResult.messageContent ? "TEXT_RESPONSE" : "EMPTY");
|
||||
console.log(` -> ${status} (HTTP ${chatResult.status}, content type: ${chatResult.messageContentType})`);
|
||||
if (extractedJson) {
|
||||
console.log(` JSON keys: ${Object.keys(extractedJson).join(", ")}`);
|
||||
} else if (chatResult.messageContent) {
|
||||
console.log(` Content preview: ${(typeof chatResult.messageContent === "string" ? chatResult.messageContent : String(chatResult.messageContent)).slice(0, 150)}...`);
|
||||
}
|
||||
|
||||
} catch (e) {
|
||||
results[model][testKey] = { error: e.message };
|
||||
console.log(` -> ERROR: ${e.message}`);
|
||||
}
|
||||
|
||||
// Generate (fallback test)
|
||||
let generateResult;
|
||||
try {
|
||||
const generatePrompt = test.system ? `${test.system}\n\n${test.user}` : test.user;
|
||||
generateResult = await runGenerate(model, generatePrompt);
|
||||
|
||||
results[model][`${testKey}_generate`] = {
|
||||
endpoint: "/api/generate",
|
||||
httpStatus: generateResult.status,
|
||||
responseFirst200: generateResult.responseFirst200,
|
||||
responseLooksLikeStructuredJSON: generateResult.responseField?.trim().startsWith("{"),
|
||||
rawPreview: generateResult.responseFirst200,
|
||||
};
|
||||
|
||||
const isJson = generateResult.responseField?.trim().startsWith("{") ? "JSON_START" : "NOT_JSON";
|
||||
console.log(` -> ${isJson} (HTTP ${generateResult.status})`);
|
||||
|
||||
} catch (e) {
|
||||
results[model][`${testKey}_generate`] = { error: e.message };
|
||||
console.log(` -> GENERATE ERROR: ${e.message}`);
|
||||
}
|
||||
|
||||
console.log();
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Save results
|
||||
// ============================================================
|
||||
|
||||
const saveFile = join(RESULTS_DIR, "debug-results.json");
|
||||
writeFileSync(saveFile, JSON.stringify(results, null, 2));
|
||||
console.log(`\nResults saved to: ${saveFile}`);
|
||||
@@ -0,0 +1,653 @@
|
||||
{
|
||||
"timestamp": "2026-07-31T18:14:22.632Z",
|
||||
"provider": "mock",
|
||||
"promptVersion": "v0.2",
|
||||
"casesRun": 35,
|
||||
"summary": {
|
||||
"schemaValidityRate": "100.0%",
|
||||
"classificationMatchRate": "28.6%",
|
||||
"reasoningModeMatchRate": "48.6%",
|
||||
"requiredConceptMatchRate": "0.0%",
|
||||
"unsupportedInferenceFailures": "0",
|
||||
"averageResponseDurationMs": "1",
|
||||
"fullPassRate": "28.6%"
|
||||
},
|
||||
"testCaseResults": [
|
||||
{
|
||||
"id": "tc-001",
|
||||
"input": "All customers cannot download their invoices.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 4,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-002",
|
||||
"input": "Some customers cannot download their invoices.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-003",
|
||||
"input": "Complaints increased by 35%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-004",
|
||||
"input": "Complaints increased by 35% while production increased by 40%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-005",
|
||||
"input": "Sales are falling.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-006",
|
||||
"input": "Sales fell sharply immediately after the price increase.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-007",
|
||||
"input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-008",
|
||||
"input": "Revenue from the premium tier dropped while total revenue grew.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-009",
|
||||
"input": "We need to improve our customer retention rate.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-010",
|
||||
"input": "The system latency went from 200ms to 5 seconds on Tuesday.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-011",
|
||||
"input": "The new release should fix the login issue.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-012",
|
||||
"input": "I think therefore I am.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-013",
|
||||
"input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-014",
|
||||
"input": "The warehouse manager reported that inventory counts don't match the system.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-015",
|
||||
"input": "We've seen a 35% increase in customer complaints.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-016",
|
||||
"input": "The number of active users increased by 500%, from 4 to 2,001.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-017",
|
||||
"input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-018",
|
||||
"input": "The manufacturing team needs better quality control.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-019",
|
||||
"input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-020",
|
||||
"input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-021",
|
||||
"input": "Production output was 1,200 units last month and 1,180 units this month.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-022",
|
||||
"input": "The CFO reported that the company's cash position is healthy.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-023",
|
||||
"input": "We have enough funding to operate for 18 months.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-024",
|
||||
"input": "The new feature was deployed at 3am and user complaints tripled the next day.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-025",
|
||||
"input": "We need to launch a mobile app to capture market share.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-026",
|
||||
"input": "The system has been running for 90 days without failure since the migration.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-027",
|
||||
"input": "No one has submitted the required compliance report despite multiple reminders.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-028",
|
||||
"input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-029",
|
||||
"input": "We should implement the new CRM because our competitors have one.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-030",
|
||||
"input": "The server response time was acceptable last quarter but degraded this month.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-031",
|
||||
"input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-032",
|
||||
"input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-033",
|
||||
"input": "The database schema was changed on Friday but the reports are still working.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-034",
|
||||
"input": "Some team members say the new process is better while others say it's slower.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-035",
|
||||
"input": "The application works fine on Chrome but not on Safari.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"investigate_contradiction"
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,652 @@
|
||||
{
|
||||
"timestamp": "2026-07-31T18:16:32.255Z",
|
||||
"provider": "mock",
|
||||
"promptVersion": "v0.2",
|
||||
"casesRun": 35,
|
||||
"summary": {
|
||||
"schemaValidityRate": "100.0%",
|
||||
"classificationMatchRate": "37.1%",
|
||||
"reasoningModeMatchRate": "71.4%",
|
||||
"requiredConceptMatchRate": "0.0%",
|
||||
"unsupportedInferenceFailures": "0",
|
||||
"averageResponseDurationMs": "1",
|
||||
"fullPassRate": "37.1%"
|
||||
},
|
||||
"testCaseResults": [
|
||||
{
|
||||
"id": "tc-001",
|
||||
"input": "All customers cannot download their invoices.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 4,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-002",
|
||||
"input": "Some customers cannot download their invoices.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-003",
|
||||
"input": "Complaints increased by 35%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-004",
|
||||
"input": "Complaints increased by 35% while production increased by 40%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-005",
|
||||
"input": "Sales are falling.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-006",
|
||||
"input": "Sales fell sharply immediately after the price increase.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-007",
|
||||
"input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-008",
|
||||
"input": "Revenue from the premium tier dropped while total revenue grew.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-009",
|
||||
"input": "We need to improve our customer retention rate.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-010",
|
||||
"input": "The system latency went from 200ms to 5 seconds on Tuesday.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-011",
|
||||
"input": "The new release should fix the login issue.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-012",
|
||||
"input": "I think therefore I am.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-013",
|
||||
"input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-014",
|
||||
"input": "The warehouse manager reported that inventory counts don't match the system.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-015",
|
||||
"input": "We've seen a 35% increase in customer complaints.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-016",
|
||||
"input": "The number of active users increased by 500%, from 4 to 2,001.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-017",
|
||||
"input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-018",
|
||||
"input": "The manufacturing team needs better quality control.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-019",
|
||||
"input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-020",
|
||||
"input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-021",
|
||||
"input": "Production output was 1,200 units last month and 1,180 units this month.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-022",
|
||||
"input": "The CFO reported that the company's cash position is healthy.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-023",
|
||||
"input": "We have enough funding to operate for 18 months.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-024",
|
||||
"input": "The new feature was deployed at 3am and user complaints tripled the next day.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-025",
|
||||
"input": "We need to launch a mobile app to capture market share.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-026",
|
||||
"input": "The system has been running for 90 days without failure since the migration.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-027",
|
||||
"input": "No one has submitted the required compliance report despite multiple reminders.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-028",
|
||||
"input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-029",
|
||||
"input": "We should implement the new CRM because our competitors have one.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-030",
|
||||
"input": "The server response time was acceptable last quarter but degraded this month.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-031",
|
||||
"input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-032",
|
||||
"input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-033",
|
||||
"input": "The database schema was changed on Friday but the reports are still working.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-034",
|
||||
"input": "Some team members say the new process is better while others say it's slower.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"investigate_contradiction",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-035",
|
||||
"input": "The application works fine on Chrome but not on Safari.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,633 @@
|
||||
{
|
||||
"timestamp": "2026-07-31T18:23:11.669Z",
|
||||
"provider": "mock",
|
||||
"promptVersion": "v0.2",
|
||||
"casesRun": 35,
|
||||
"summary": {
|
||||
"schemaValidityRate": "100.0%",
|
||||
"classificationMatchRate": "48.6%",
|
||||
"reasoningModeMatchRate": "48.6%",
|
||||
"requiredConceptMatchRate": "0.0%",
|
||||
"unsupportedInferenceFailures": "0",
|
||||
"averageResponseDurationMs": "1",
|
||||
"fullPassRate": "48.6%"
|
||||
},
|
||||
"testCaseResults": [
|
||||
{
|
||||
"id": "tc-001",
|
||||
"input": "All customers cannot download their invoices.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 6,
|
||||
"actualPrimaryType": "fault_report",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-002",
|
||||
"input": "Some customers cannot download their invoices.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-003",
|
||||
"input": "Complaints increased by 35%.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "fault_report",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-004",
|
||||
"input": "Complaints increased by 35% while production increased by 40%.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "fault_report",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-005",
|
||||
"input": "Sales are falling.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "fault_report",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-006",
|
||||
"input": "Sales fell sharply immediately after the price increase.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-007",
|
||||
"input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-008",
|
||||
"input": "Revenue from the premium tier dropped while total revenue grew.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-009",
|
||||
"input": "We need to improve our customer retention rate.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-010",
|
||||
"input": "The system latency went from 200ms to 5 seconds on Tuesday.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-011",
|
||||
"input": "The new release should fix the login issue.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-012",
|
||||
"input": "I think therefore I am.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-013",
|
||||
"input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-014",
|
||||
"input": "The warehouse manager reported that inventory counts don't match the system.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-015",
|
||||
"input": "We've seen a 35% increase in customer complaints.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "fault_report",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-016",
|
||||
"input": "The number of active users increased by 500%, from 4 to 2,001.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-017",
|
||||
"input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-018",
|
||||
"input": "The manufacturing team needs better quality control.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-019",
|
||||
"input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "fault_report",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-020",
|
||||
"input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-021",
|
||||
"input": "Production output was 1,200 units last month and 1,180 units this month.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-022",
|
||||
"input": "The CFO reported that the company's cash position is healthy.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-023",
|
||||
"input": "We have enough funding to operate for 18 months.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-024",
|
||||
"input": "The new feature was deployed at 3am and user complaints tripled the next day.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "fault_report",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-025",
|
||||
"input": "We need to launch a mobile app to capture market share.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-026",
|
||||
"input": "The system has been running for 90 days without failure since the migration.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-027",
|
||||
"input": "No one has submitted the required compliance report despite multiple reminders.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "fault_report",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-028",
|
||||
"input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-029",
|
||||
"input": "We should implement the new CRM because our competitors have one.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-030",
|
||||
"input": "The server response time was acceptable last quarter but degraded this month.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-031",
|
||||
"input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "fault_report",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-032",
|
||||
"input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-033",
|
||||
"input": "The database schema was changed on Friday but the reports are still working.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-034",
|
||||
"input": "Some team members say the new process is better while others say it's slower.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"investigate_contradiction",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-035",
|
||||
"input": "The application works fine on Chrome but not on Safari.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "other",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,639 @@
|
||||
{
|
||||
"timestamp": "2026-07-31T18:34:17.855Z",
|
||||
"provider": "mock",
|
||||
"promptVersion": "v0.2",
|
||||
"casesRun": 35,
|
||||
"summary": {
|
||||
"schemaValidityRate": "100.0%",
|
||||
"classificationMatchRate": "82.9%",
|
||||
"reasoningModeMatchRate": "51.4%",
|
||||
"requiredConceptMatchRate": "0.0%",
|
||||
"unsupportedInferenceFailures": "0",
|
||||
"averageResponseDurationMs": "1",
|
||||
"fullPassRate": "82.9%"
|
||||
},
|
||||
"testCaseResults": [
|
||||
{
|
||||
"id": "tc-001",
|
||||
"input": "All customers cannot download their invoices.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 5,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-002",
|
||||
"input": "Some customers cannot download their invoices.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 2,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-003",
|
||||
"input": "Complaints increased by 35%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-004",
|
||||
"input": "Complaints increased by 35% while production increased by 40%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-005",
|
||||
"input": "Sales are falling.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-006",
|
||||
"input": "Sales fell sharply immediately after the price increase.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-007",
|
||||
"input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"investigate_contradiction",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-008",
|
||||
"input": "Revenue from the premium tier dropped while total revenue grew.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-009",
|
||||
"input": "We need to improve our customer retention rate.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-010",
|
||||
"input": "The system latency went from 200ms to 5 seconds on Tuesday.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-011",
|
||||
"input": "The new release should fix the login issue.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-012",
|
||||
"input": "I think therefore I am.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-013",
|
||||
"input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-014",
|
||||
"input": "The warehouse manager reported that inventory counts don't match the system.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-015",
|
||||
"input": "We've seen a 35% increase in customer complaints.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-016",
|
||||
"input": "The number of active users increased by 500%, from 4 to 2,001.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-017",
|
||||
"input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-018",
|
||||
"input": "The manufacturing team needs better quality control.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-019",
|
||||
"input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-020",
|
||||
"input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-021",
|
||||
"input": "Production output was 1,200 units last month and 1,180 units this month.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-022",
|
||||
"input": "The CFO reported that the company's cash position is healthy.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-023",
|
||||
"input": "We have enough funding to operate for 18 months.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-024",
|
||||
"input": "The new feature was deployed at 3am and user complaints tripled the next day.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-025",
|
||||
"input": "We need to launch a mobile app to capture market share.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-026",
|
||||
"input": "The system has been running for 90 days without failure since the migration.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-027",
|
||||
"input": "No one has submitted the required compliance report despite multiple reminders.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-028",
|
||||
"input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-029",
|
||||
"input": "We should implement the new CRM because our competitors have one.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-030",
|
||||
"input": "The server response time was acceptable last quarter but degraded this month.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-031",
|
||||
"input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-032",
|
||||
"input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-033",
|
||||
"input": "The database schema was changed on Friday but the reports are still working.",
|
||||
"pass": false,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": false,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-034",
|
||||
"input": "Some team members say the new process is better while others say it's slower.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-035",
|
||||
"input": "The application works fine on Chrome but not on Safari.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,644 @@
|
||||
{
|
||||
"timestamp": "2026-07-31T18:59:37.227Z",
|
||||
"provider": "mock",
|
||||
"promptVersion": "v0.2",
|
||||
"casesRun": 35,
|
||||
"summary": {
|
||||
"schemaValidityRate": "100.0%",
|
||||
"classificationMatchRate": "100.0%",
|
||||
"reasoningModeMatchRate": "57.1%",
|
||||
"requiredConceptMatchRate": "0.0%",
|
||||
"unsupportedInferenceFailures": "0",
|
||||
"averageResponseDurationMs": "0",
|
||||
"fullPassRate": "100.0%"
|
||||
},
|
||||
"testCaseResults": [
|
||||
{
|
||||
"id": "tc-001",
|
||||
"input": "All customers cannot download their invoices.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 4,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-002",
|
||||
"input": "Some customers cannot download their invoices.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-003",
|
||||
"input": "Complaints increased by 35%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-004",
|
||||
"input": "Complaints increased by 35% while production increased by 40%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-005",
|
||||
"input": "Sales are falling.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-006",
|
||||
"input": "Sales fell sharply immediately after the price increase.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-007",
|
||||
"input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"investigate_contradiction",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-008",
|
||||
"input": "Revenue from the premium tier dropped while total revenue grew.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-009",
|
||||
"input": "We need to improve our customer retention rate.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-010",
|
||||
"input": "The system latency went from 200ms to 5 seconds on Tuesday.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-011",
|
||||
"input": "The new release should fix the login issue.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-012",
|
||||
"input": "I think therefore I am.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-013",
|
||||
"input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-014",
|
||||
"input": "The warehouse manager reported that inventory counts don't match the system.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-015",
|
||||
"input": "We've seen a 35% increase in customer complaints.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-016",
|
||||
"input": "The number of active users increased by 500%, from 4 to 2,001.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-017",
|
||||
"input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"investigate_contradiction",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-018",
|
||||
"input": "The manufacturing team needs better quality control.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-019",
|
||||
"input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-020",
|
||||
"input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-021",
|
||||
"input": "Production output was 1,200 units last month and 1,180 units this month.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-022",
|
||||
"input": "The CFO reported that the company's cash position is healthy.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-023",
|
||||
"input": "We have enough funding to operate for 18 months.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-024",
|
||||
"input": "The new feature was deployed at 3am and user complaints tripled the next day.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-025",
|
||||
"input": "We need to launch a mobile app to capture market share.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-026",
|
||||
"input": "The system has been running for 90 days without failure since the migration.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-027",
|
||||
"input": "No one has submitted the required compliance report despite multiple reminders.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-028",
|
||||
"input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-029",
|
||||
"input": "We should implement the new CRM because our competitors have one.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-030",
|
||||
"input": "The server response time was acceptable last quarter but degraded this month.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-031",
|
||||
"input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-032",
|
||||
"input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-033",
|
||||
"input": "The database schema was changed on Friday but the reports are still working.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-034",
|
||||
"input": "Some team members say the new process is better while others say it's slower.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-035",
|
||||
"input": "The application works fine on Chrome but not on Safari.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,644 @@
|
||||
{
|
||||
"timestamp": "2026-07-31T19:00:09.575Z",
|
||||
"provider": "mock",
|
||||
"promptVersion": "v0.2",
|
||||
"casesRun": 35,
|
||||
"summary": {
|
||||
"schemaValidityRate": "100.0%",
|
||||
"classificationMatchRate": "100.0%",
|
||||
"reasoningModeMatchRate": "57.1%",
|
||||
"requiredConceptMatchRate": "0.0%",
|
||||
"unsupportedInferenceFailures": "0",
|
||||
"averageResponseDurationMs": "1",
|
||||
"fullPassRate": "100.0%"
|
||||
},
|
||||
"testCaseResults": [
|
||||
{
|
||||
"id": "tc-001",
|
||||
"input": "All customers cannot download their invoices.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 6,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-002",
|
||||
"input": "Some customers cannot download their invoices.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 2,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-003",
|
||||
"input": "Complaints increased by 35%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-004",
|
||||
"input": "Complaints increased by 35% while production increased by 40%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-005",
|
||||
"input": "Sales are falling.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-006",
|
||||
"input": "Sales fell sharply immediately after the price increase.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-007",
|
||||
"input": "The quarterly revenue exceeded targets but net profit declined by 12%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"investigate_contradiction",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-008",
|
||||
"input": "Revenue from the premium tier dropped while total revenue grew.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-009",
|
||||
"input": "We need to improve our customer retention rate.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-010",
|
||||
"input": "The system latency went from 200ms to 5 seconds on Tuesday.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-011",
|
||||
"input": "The new release should fix the login issue.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-012",
|
||||
"input": "I think therefore I am.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-013",
|
||||
"input": "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "ambiguous_statement",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"clarify_meaning"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-014",
|
||||
"input": "The warehouse manager reported that inventory counts don't match the system.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-015",
|
||||
"input": "We've seen a 35% increase in customer complaints.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-016",
|
||||
"input": "The number of active users increased by 500%, from 4 to 2,001.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-017",
|
||||
"input": "User engagement metrics improved but the support ticket backlog grew by 200%.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "contradiction",
|
||||
"actualReasoningModes": [
|
||||
"investigate_contradiction",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-018",
|
||||
"input": "The manufacturing team needs better quality control.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-019",
|
||||
"input": "All users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-020",
|
||||
"input": "Some users in the EU region are getting a 403 error when trying to access the dashboard.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-021",
|
||||
"input": "Production output was 1,200 units last month and 1,180 units this month.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-022",
|
||||
"input": "The CFO reported that the company's cash position is healthy.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "reported_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"validate_claim"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-023",
|
||||
"input": "We have enough funding to operate for 18 months.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-024",
|
||||
"input": "The new feature was deployed at 3am and user complaints tripled the next day.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"establish_baseline",
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-025",
|
||||
"input": "We need to launch a mobile app to capture market share.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-026",
|
||||
"input": "The system has been running for 90 days without failure since the migration.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-027",
|
||||
"input": "No one has submitted the required compliance report despite multiple reminders.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-028",
|
||||
"input": "The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-029",
|
||||
"input": "We should implement the new CRM because our competitors have one.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "decision_request",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"decision_support",
|
||||
"identify_missing_information"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-030",
|
||||
"input": "The server response time was acceptable last quarter but degraded this month.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-031",
|
||||
"input": "The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-032",
|
||||
"input": "External analysts expect our industry to decline by 15% next year due to regulatory changes.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "causal_claim",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-033",
|
||||
"input": "The database schema was changed on Friday but the reports are still working.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "unexplained_change",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference",
|
||||
"establish_baseline",
|
||||
"validate_measurement"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-034",
|
||||
"input": "Some team members say the new process is better while others say it's slower.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": false,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 1,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
},
|
||||
{
|
||||
"id": "tc-035",
|
||||
"input": "The application works fine on Chrome but not on Safari.",
|
||||
"pass": true,
|
||||
"schemaValid": true,
|
||||
"classificationMatch": true,
|
||||
"reasoningModeMatch": true,
|
||||
"requiredConceptMatch": false,
|
||||
"unsupportedInferenceAbsence": true,
|
||||
"nextQuestionPresent": true,
|
||||
"responseDurationMs": 0,
|
||||
"actualPrimaryType": "observed_problem",
|
||||
"actualReasoningModes": [
|
||||
"identify_difference"
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,92 @@
|
||||
[
|
||||
{
|
||||
"id": "diag-01",
|
||||
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
||||
"expectedPrimaryTypes": ["unexplained_change"],
|
||||
"expectedReasoningModes": ["establish_baseline", "identify_difference"],
|
||||
"shouldIdentify": ["complaints", "warehouse", "baseline comparison"],
|
||||
"shouldNotInfer": ["quality issue", "staff turnover", "training gap"],
|
||||
"description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues."
|
||||
},
|
||||
{
|
||||
"id": "diag-02",
|
||||
"input": "Some customers reported that the new app crashes when uploading photos.",
|
||||
"expectedPrimaryTypes": ["observed_problem"],
|
||||
"expectedReasoningModes": ["identify_difference", "establish_baseline"],
|
||||
"shouldIdentify": ["app crashes", "photo upload", "some customers"],
|
||||
"shouldNotInfer": ["all users affected", "server-side bug", "Android only"],
|
||||
"description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims."
|
||||
},
|
||||
{
|
||||
"id": "diag-03",
|
||||
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
||||
"expectedPrimaryTypes": ["contradiction"],
|
||||
"expectedReasoningModes": ["investigate_contradiction", "establish_baseline"],
|
||||
"shouldIdentify": ["sales decline", "price increase", "revenue increase", "CFO report"],
|
||||
"shouldNotInfer": ["price was set too high", "competitors gained market share", "revenue data is wrong"],
|
||||
"description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value."
|
||||
},
|
||||
{
|
||||
"id": "diag-04",
|
||||
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
||||
"expectedPrimaryTypes": ["decision_request"],
|
||||
"expectedReasoningModes": ["decision_support", "identify_missing_information"],
|
||||
"shouldIdentify": ["marketplace app", "Southeast Asia", "competitor gap"],
|
||||
"shouldNotInfer": ["this will definitely succeed", "we have the resources", "competitors are struggling"],
|
||||
"description": "Decision request — forward-looking, needs missing info identification."
|
||||
},
|
||||
{
|
||||
"id": "diag-05",
|
||||
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
||||
"expectedPrimaryTypes": ["unexplained_change"],
|
||||
"expectedReasoningModes": ["establish_baseline", "identify_difference"],
|
||||
"shouldIdentify": ["supplier change", "three months ago", "same defect rate"],
|
||||
"shouldNotInfer": ["new supplier is worse", "old supplier was better", "quality process is broken"],
|
||||
"description": "Unexpected continuity — changed context but no outcome change."
|
||||
},
|
||||
{
|
||||
"id": "diag-06",
|
||||
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
||||
"expectedPrimaryTypes": ["unexplained_change"],
|
||||
"expectedReasoningModes": ["establish_baseline", "validate_measurement"],
|
||||
"shouldIdentify": ["completion rate", "45%", "62%", "onboarding"],
|
||||
"shouldNotInfer": ["all improvements are due to the redesign", "the old flow was bad", "users prefer the new design"],
|
||||
"description": "Quantified improvement — needs context about measurement period and baseline conditions."
|
||||
},
|
||||
{
|
||||
"id": "diag-07",
|
||||
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
||||
"expectedPrimaryTypes": ["reported_claim"],
|
||||
"expectedReasoningModes": ["validate_claim", "identify_difference"],
|
||||
"shouldIdentify": ["pricing complexity", "small business", "user claim"],
|
||||
"shouldNotInfer": ["the pricing is actually complex", "other small businesses agree", "we should simplify pricing"],
|
||||
"description": "Single reported claim — needs validation, not acceptance as fact."
|
||||
},
|
||||
{
|
||||
"id": "diag-08",
|
||||
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
||||
"expectedPrimaryTypes": ["ambiguous_statement"],
|
||||
"expectedReasoningModes": ["clarify_meaning"],
|
||||
"shouldIdentify": ["philosophical", "ambiguous", "meaning clarification"],
|
||||
"shouldNotInfer": ["the phrase was wrong", "the colleague is hostile", "we should avoid philosophical language"],
|
||||
"description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode."
|
||||
},
|
||||
{
|
||||
"id": "diag-09",
|
||||
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
||||
"expectedPrimaryTypes": ["causal_claim"],
|
||||
"expectedReasoningModes": ["investigate_contradiction", "establish_baseline"],
|
||||
"shouldIdentify": ["deployment", "complaint volume increase", "tripled", "47 cases"],
|
||||
"shouldNotInfer": ["the deployment caused the complaints", "the bug report was insufficient", "rollback is needed"],
|
||||
"description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion."
|
||||
},
|
||||
{
|
||||
"id": "diag-10",
|
||||
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
||||
"expectedPrimaryTypes": ["observed_problem"],
|
||||
"expectedReasoningModes": ["identify_difference", "decompose_aggregate"],
|
||||
"shouldIdentify": ["production issues", "delivery speed", "complaint types"],
|
||||
"shouldNotInfer": ["production is worse than delivery", "the delivery team needs training", "both teams are underperforming equally"],
|
||||
"description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims."
|
||||
}
|
||||
]
|
||||
+565
-121
@@ -1,8 +1,23 @@
|
||||
import { describe, it, expect, vi } from "vitest";
|
||||
import { reconstructionSchema } from "@/lib/reconstruction/schema";
|
||||
import { parseReconstruction } from "@/lib/reconstruction/schema";
|
||||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
reconstructionSchema,
|
||||
confidenceEnum,
|
||||
importanceEnum,
|
||||
inputTypes,
|
||||
reasoningModes,
|
||||
evidenceRecordSchema,
|
||||
reconstructionV2Schema,
|
||||
analyseResponseSchema,
|
||||
parseReconstruction,
|
||||
parseReconstructionV2,
|
||||
} from "@/lib/reconstruction/schema";
|
||||
import { CONFIDENCE_VALUES } from "@/lib/llm/types.js";
|
||||
|
||||
describe("reconstruction schema", () => {
|
||||
// ──────────────────────────────────────────────
|
||||
// v0.1 — backward compatibility tests
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("v0.1 reconstruction schema", () => {
|
||||
it("validates a complete valid reconstruction", () => {
|
||||
const input = {
|
||||
observations: [{ id: "o1", description: "Saw smoke", confidence: "high" }],
|
||||
@@ -23,34 +38,19 @@ describe("reconstruction schema", () => {
|
||||
it("rejects invalid confidence values", () => {
|
||||
const input = {
|
||||
observations: [{ id: "o1", description: "test", confidence: "extreme" }],
|
||||
reportedClaims: [],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
transitions: [],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
reportedClaims: [], assumptions: [], entities: [], transitions: [],
|
||||
expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
};
|
||||
|
||||
const result = reconstructionSchema.safeParse(input);
|
||||
expect(result.success).toBe(false);
|
||||
if (!result.success) {
|
||||
expect(result.error.issues[0].message).toContain("Expected");
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects missing required fields", () => {
|
||||
const input = {
|
||||
observations: [{ id: "o1" }],
|
||||
reportedClaims: [],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
transitions: [],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
reportedClaims: [], assumptions: [], entities: [], transitions: [],
|
||||
expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
};
|
||||
|
||||
const result = reconstructionSchema.safeParse(input);
|
||||
@@ -61,13 +61,7 @@ describe("reconstruction schema", () => {
|
||||
const input = {
|
||||
observations: [],
|
||||
reportedClaims: [{ id: "rc1", description: "test", confidence: "very_high", attributedTo: null }],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
transitions: [],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
};
|
||||
|
||||
const result = reconstructionSchema.safeParse(input);
|
||||
@@ -76,15 +70,9 @@ describe("reconstruction schema", () => {
|
||||
|
||||
it("rejects empty transitions", () => {
|
||||
const input = {
|
||||
observations: [],
|
||||
reportedClaims: [],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
observations: [], reportedClaims: [], assumptions: [], entities: [],
|
||||
transitions: [{ id: "t1", description: "", confidence: "high", entity: "", previousState: "", currentState: "", explanationStatus: "" }],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
};
|
||||
|
||||
const result = reconstructionSchema.safeParse(input);
|
||||
@@ -95,13 +83,7 @@ describe("reconstruction schema", () => {
|
||||
const input = {
|
||||
observations: [],
|
||||
reportedClaims: [{ id: "rc1", description: "Someone called it in", confidence: "medium", attributedTo: null }],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
transitions: [],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
};
|
||||
|
||||
const result = reconstructionSchema.safeParse(input);
|
||||
@@ -109,18 +91,255 @@ describe("reconstruction schema", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseReconstruction", () => {
|
||||
// ──────────────────────────────────────────────
|
||||
// v0.2 — schema validation tests
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("v0.2 input classification", () => {
|
||||
it.each([
|
||||
"observed_problem", "unexplained_change", "contradiction", "decision_request",
|
||||
"causal_claim", "reported_claim", "fault_report", "ambiguous_statement",
|
||||
"question", "desired_outcome", "insufficient_context", "other",
|
||||
])("validates input type '%s'", (type) => {
|
||||
const result = inputTypes.safeParse(type);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects invalid input types", () => {
|
||||
expect(inputTypes.safeParse("invalid_type").success).toBe(false);
|
||||
expect(inputTypes.safeParse("").success).toBe(false);
|
||||
expect(inputTypes.safeParse(null).success).toBe(false);
|
||||
});
|
||||
|
||||
it("validates reasoning modes", () => {
|
||||
const modes = [
|
||||
"establish_baseline", "identify_difference", "reconstruct_transition",
|
||||
"decompose_aggregate", "validate_measurement", "validate_claim",
|
||||
"investigate_contradiction", "clarify_meaning", "decision_support",
|
||||
"fault_investigation", "identify_missing_information", "test_possible_explanations", "other",
|
||||
];
|
||||
for (const m of modes) {
|
||||
const result = reasoningModes.safeParse(m);
|
||||
expect(result.success).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects invalid reasoning mode", () => {
|
||||
expect(reasoningModes.safeParse("no_op").success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("v0.2 multiple secondary types and reasoning modes", () => {
|
||||
it("validates classification with multiple secondary types", () => {
|
||||
const classification = {
|
||||
primaryType: "observed_problem",
|
||||
secondaryTypes: ["fault_report", "decision_request"],
|
||||
reasoningModes: ["validate_claim", "identify_missing_information"],
|
||||
classificationReason: "Test scenario with multiple classifications",
|
||||
confidence: "high",
|
||||
};
|
||||
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: classification,
|
||||
reconstruction: {
|
||||
summary: "test", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [],
|
||||
differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [],
|
||||
importantUnknowns: [], plausibleInterpretations: [],
|
||||
},
|
||||
evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "supporting" }],
|
||||
nextQuestion: {
|
||||
id: "q1", question: "Test?", targets: ["x"], reason: "r",
|
||||
expectedInformationValue: "medium", reasoningMode: "other",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("validates with single secondary type", () => {
|
||||
const classification = {
|
||||
primaryType: "unexplained_change",
|
||||
secondaryTypes: ["observed_problem"],
|
||||
reasoningModes: ["establish_baseline"],
|
||||
classificationReason: "Single secondary",
|
||||
confidence: "medium",
|
||||
};
|
||||
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: classification,
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "medium", importance: "supporting" }],
|
||||
nextQuestion: { id: "q1", question: "Test?", targets: ["x"], reason: "r", expectedInformationValue: "low", reasoningMode: "other" },
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("validates with multiple reasoning modes", () => {
|
||||
const classification = {
|
||||
primaryType: "contradiction",
|
||||
secondaryTypes: [],
|
||||
reasoningModes: ["investigate_contradiction", "identify_difference", "validate_claim"],
|
||||
classificationReason: "Multiple reasoning modes applicable",
|
||||
confidence: "high",
|
||||
};
|
||||
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: classification,
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "important" }],
|
||||
nextQuestion: { id: "q1", question: "Test?", targets: ["x"], reason: "r", expectedInformationValue: "high", reasoningMode: "investigate_contradiction" },
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("v0.2 evidence records", () => {
|
||||
it.each([
|
||||
"direct_observation", "reported_statement", "interpretation", "assumption", "inferred_relationship",
|
||||
])("validates evidence type '%s'", (eType) => {
|
||||
const result = evidenceRecordSchema.safeParse({
|
||||
id: "e1", description: "test", evidenceType: eType, confidence: "high", importance: "supporting",
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects invalid evidence type", () => {
|
||||
const result = evidenceRecordSchema.safeParse({
|
||||
id: "e1", description: "test", evidenceType: "unknown_type", confidence: "high", importance: "supporting",
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("allows null attribution", () => {
|
||||
const result = evidenceRecordSchema.safeParse({
|
||||
id: "e1", description: "test", evidenceType: "reported_statement",
|
||||
attribution: null, confidence: "medium", importance: "incidental",
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("requires source or attribution optional but not both mandatory", () => {
|
||||
const result = evidenceRecordSchema.safeParse({
|
||||
id: "e1", description: "test", evidenceType: "direct_observation",
|
||||
confidence: "high", importance: "critical",
|
||||
});
|
||||
expect(result.success).toBe(true); // source and attribution are optional
|
||||
});
|
||||
});
|
||||
|
||||
describe("v0.2 invalid confidence and importance values", () => {
|
||||
it.each(["very_high", "extreme", "low_medium", "", "null"])(
|
||||
"invalid confidence '%s' rejected", (val) => {
|
||||
const result = confidenceEnum.safeParse(val);
|
||||
expect(result.success).toBe(false);
|
||||
}
|
||||
);
|
||||
|
||||
it("valid confidence values accepted", () => {
|
||||
for (const v of ["low", "medium", "high"]) {
|
||||
const result = confidenceEnum.safeParse(v);
|
||||
expect(result.success).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["very_high", "extreme", "low_medium", "", "critical_plus"])(
|
||||
"invalid importance '%s' rejected", (val) => {
|
||||
const result = evidenceRecordSchema.safeParse({
|
||||
id: "e1", description: "test", evidenceType: "direct_observation",
|
||||
confidence: "high", importance: val,
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
}
|
||||
);
|
||||
|
||||
it.each(["incidental", "supporting", "important", "critical"])(
|
||||
"valid importance '%s' accepted", (val) => {
|
||||
const result = evidenceRecordSchema.safeParse({
|
||||
id: "e1", description: "test", evidenceType: "direct_observation",
|
||||
confidence: "high", importance: val,
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
describe("v0.2 plausible interpretations", () => {
|
||||
it("validates reconstruction with multiple plausible interpretations", () => {
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: {
|
||||
primaryType: "decision_support", secondaryTypes: [], reasoningModes: [],
|
||||
classificationReason: "Multiple interpretations possible.", confidence: "medium",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: "The situation has two competing explanations.",
|
||||
actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [],
|
||||
knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [],
|
||||
plausibleInterpretations: [
|
||||
{
|
||||
id: "pi1", description: "The issue is caused by configuration drift",
|
||||
supportingEvidenceIds: ["e1", "e3"], assumptionsRequired: ["config_history_is_incomplete"], confidence: "medium",
|
||||
},
|
||||
{
|
||||
id: "pi2", description: "The issue stems from upstream dependency failure",
|
||||
supportingEvidenceIds: ["e2"], assumptionsRequired: ["dependency_outage_at_same_time"], confidence: "low",
|
||||
},
|
||||
],
|
||||
},
|
||||
evidence: [{ id: "e1", description: "Config changed on Tuesday", evidenceType: "direct_observation", confidence: "high", importance: "supporting" }],
|
||||
nextQuestion: { id: "q1", question: "What changed between Monday and Tuesday?", targets: ["timeline"], reason: "To distinguish between drift and dependency failure.", expectedInformationValue: "high", reasoningMode: "reconstruct_transition" },
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("allows interpretation with empty assumptionsRequired", () => {
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: { primaryType: "other", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "low" },
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [{ id: "pi1", description: "Plain interpretation", supportingEvidenceIds: ["e1"], confidence: "low" }] },
|
||||
evidence: [], nextQuestion: { id: "q1", question: "?", targets: [], reason: "r", expectedInformationValue: "low", reasoningMode: "other" },
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("v0.2 exactly one next question", () => {
|
||||
it("validates when exactly one next question is present", () => {
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: { primaryType: "observed_problem", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "high" },
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [], nextQuestion: { id: "q1", question: "What is the baseline?", targets: ["baseline"], reason: "r", expectedInformationValue: "high", reasoningMode: "establish_baseline" },
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("validates when nextQuestion is absent (schema allows optional)", () => {
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: { primaryType: "ambiguous_statement", secondaryTypes: [], reasoningModes: [], classificationReason: "No question possible.", confidence: "low" },
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [], nextQuestion: undefined,
|
||||
});
|
||||
|
||||
// The schema allows missing nextQuestion (optional), so this should pass validation.
|
||||
// We validate exactly-one at the evaluator level, not in the schema.
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects reconstructionV2 when required fields are missing", () => {
|
||||
const result = reconstructionV2Schema.safeParse({});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseReconstruction (v0.1)", () => {
|
||||
it("parses a raw JSON string", () => {
|
||||
const raw = JSON.stringify({
|
||||
observations: [{ id: "o1", description: "test", confidence: "high" }],
|
||||
reportedClaims: [],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
transitions: [],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
reportedClaims: [], assumptions: [], entities: [], transitions: [],
|
||||
expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
});
|
||||
|
||||
const result = parseReconstruction(raw);
|
||||
@@ -134,18 +353,119 @@ describe("parseReconstruction", () => {
|
||||
it("rejects valid JSON that fails schema validation", () => {
|
||||
const raw = JSON.stringify({
|
||||
observations: [{ id: "o1", description: "test", confidence: "extreme" }],
|
||||
reportedClaims: [],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
transitions: [],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
reportedClaims: [], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
});
|
||||
|
||||
expect(() => parseReconstruction(raw)).toThrow();
|
||||
});
|
||||
|
||||
it("accepts an already-parsed object", () => {
|
||||
const obj = {
|
||||
observations: [{ id: "o1", description: "test", confidence: "high" }],
|
||||
reportedClaims: [], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
};
|
||||
|
||||
const result = parseReconstruction(obj);
|
||||
expect(result.observations[0].id).toBe("o1");
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseReconstructionV2", () => {
|
||||
it("parses a raw JSON v0.2 string", () => {
|
||||
const raw = JSON.stringify({
|
||||
inputClassification: { primaryType: "observed_problem", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "high" },
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "supporting" }],
|
||||
nextQuestion: { id: "q1", question: "Test?", targets: ["x"], reason: "r", expectedInformationValue: "medium", reasoningMode: "other" },
|
||||
});
|
||||
|
||||
const result = parseReconstructionV2(raw);
|
||||
expect(result.inputClassification.primaryType).toBe("observed_problem");
|
||||
});
|
||||
|
||||
it("rejects malformed JSON string", () => {
|
||||
expect(() => parseReconstructionV2("{invalid json")).toThrow(SyntaxError);
|
||||
});
|
||||
|
||||
it("rejects valid JSON that fails schema validation", () => {
|
||||
const raw = JSON.stringify({ not: "the right structure" });
|
||||
expect(() => parseReconstructionV2(raw)).toThrow();
|
||||
});
|
||||
|
||||
it("accepts an already-parsed v0.2 object", () => {
|
||||
const obj = {
|
||||
inputClassification: { primaryType: "observed_problem", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "high" },
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [], nextQuestion: undefined,
|
||||
};
|
||||
|
||||
const result = parseReconstructionV2(obj);
|
||||
expect(result.inputClassification.primaryType).toBe("observed_problem");
|
||||
});
|
||||
});
|
||||
|
||||
describe("malformed model output", () => {
|
||||
it("throws on non-JSON string", () => {
|
||||
expect(() => parseReconstruction("hello world")).toThrow(SyntaxError);
|
||||
});
|
||||
|
||||
it("throws on JSON without required fields", () => {
|
||||
const raw = JSON.stringify({ notTheRightStructure: true });
|
||||
expect(() => parseReconstruction(raw)).toThrow();
|
||||
});
|
||||
|
||||
it("handles empty arrays for all v0.1 categories", () => {
|
||||
const result = parseReconstruction({
|
||||
observations: [], reportedClaims: [], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
});
|
||||
|
||||
expect(result.observations.length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// v0.2 full reconstruction validation
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("v0.2 complete valid reconstruction", () => {
|
||||
it("validates a full v0.2 output with all sections", () => {
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: { primaryType: "observed_problem", secondaryTypes: ["fault_report"], reasoningModes: ["validate_claim", "identify_difference"], classificationReason: "Clear operational issue identified.", confidence: "high" },
|
||||
reconstruction: {
|
||||
summary: "A fault report with subset scope affecting specific users.",
|
||||
actors: [{ id: "a1", description: "Affected user group", confidence: "medium" }],
|
||||
systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [{ id: "d1", description: "Subset vs universal access", confidence: "high" }],
|
||||
knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [{ id: "u1", description: "Root cause of access failure", confidence: "medium" }],
|
||||
plausibleInterpretations: [],
|
||||
},
|
||||
evidence: [{ id: "e1", description: "User reports confirm the issue.", evidenceType: "reported_statement", source: "support tickets", confidence: "high", importance: "important" }],
|
||||
nextQuestion: { id: "q1", question: "Which specific users are affected?", targets: ["user_segment"], reason: "Narrow scope to identify pattern.", expectedInformationValue: "high", reasoningMode: "validate_claim" },
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("allows null source in evidence", () => {
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: { primaryType: "other", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "low" },
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", attribution: null, confidence: "low", importance: "incidental" }],
|
||||
nextQuestion: undefined,
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("requires all critical importance values for evidence", () => {
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: { primaryType: "other", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "low" },
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "critical" }],
|
||||
nextQuestion: { id: "q1", question: "?", targets: ["x"], reason: "r", expectedInformationValue: "low", reasoningMode: "other" },
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true); // critical importance is valid
|
||||
});
|
||||
});
|
||||
|
||||
describe("empty scenario rejection", () => {
|
||||
@@ -160,74 +480,198 @@ describe("empty scenario rejection", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Provider parsing tests
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("provider response parsing", () => {
|
||||
it("handles Ollama generate response shape", async () => {
|
||||
vi.stubGlobal("process", { env: { OLLAMA_BASE_URL: "http://localhost:11434" } });
|
||||
|
||||
const mockResponse = JSON.stringify({
|
||||
observations: [{ id: "o1", description: "test", confidence: "high" }],
|
||||
reportedClaims: [],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
transitions: [],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
});
|
||||
|
||||
global.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ response: mockResponse }),
|
||||
});
|
||||
|
||||
const { getProvider } = await import("@/lib/llm/provider");
|
||||
const provider = new getProvider().constructor ? null : getProvider();
|
||||
|
||||
// The provider is instantiated in getProvider
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
|
||||
it("handles raw JSON object response", () => {
|
||||
const parsed = parseReconstruction({
|
||||
observations: [],
|
||||
reportedClaims: [{ id: "rc1", description: "he said", confidence: "medium", attributedTo: "Alice" }],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
transitions: [],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
observations: [], reportedClaims: [{ id: "rc1", description: "he said", confidence: "medium", attributedTo: "Alice" }], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
});
|
||||
|
||||
expect(parsed.reportedClaims[0].attributedTo).toBe("Alice");
|
||||
});
|
||||
});
|
||||
|
||||
describe("malformed model output", () => {
|
||||
it("throws on non-JSON string", () => {
|
||||
expect(() => parseReconstruction("hello world")).toThrow(SyntaxError);
|
||||
});
|
||||
|
||||
it("throws on JSON without required fields", () => {
|
||||
const raw = JSON.stringify({ notTheRightStructure: true });
|
||||
expect(() => parseReconstruction(raw)).toThrow();
|
||||
});
|
||||
|
||||
it("handles empty arrays for all categories", () => {
|
||||
const result = parseReconstruction({
|
||||
observations: [],
|
||||
reportedClaims: [],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
transitions: [],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
it("handles v0.2 parsed reconstruction", () => {
|
||||
const parsed = parseReconstructionV2({
|
||||
inputClassification: { primaryType: "observed_problem", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "high" },
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [{ id: "d1", description: "delta", confidence: "high" }], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "supporting" }],
|
||||
nextQuestion: { id: "q1", question: "?", targets: ["x"], reason: "r", expectedInformationValue: "medium", reasoningMode: "other" },
|
||||
});
|
||||
|
||||
expect(result.observations.length).toBe(0);
|
||||
expect(parsed.inputClassification.primaryType).toBe("observed_problem");
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Deterministic evaluator scoring tests
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("deterministic evaluator scoring", () => {
|
||||
function normalise(text) {
|
||||
return String(text).toLowerCase().replace(/[^\w\s_]/g, " ").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function checkPrimaryTypeMatch(actualPrimary, expectedTypes) {
|
||||
if (!actualPrimary || !expectedTypes?.length) return false;
|
||||
const actual = String(actualPrimary).toLowerCase().replace(/\s+/g, "_");
|
||||
return expectedTypes.some((t) => t.toLowerCase().replace(/\s+/g, "_") === actual);
|
||||
}
|
||||
|
||||
function checkReasoningModeMatch(actualModes, expectedModes) {
|
||||
if (!actualModes?.length || !expectedModes?.length) return false;
|
||||
const actual = actualModes.map((m) => String(m).toLowerCase().replace(/\s+/g, "_"));
|
||||
const expected = expectedModes.map((m) => String(m).toLowerCase().replace(/\s+/g, "_"));
|
||||
return expected.some((e) => actual.includes(e));
|
||||
}
|
||||
|
||||
it("matches primary type when exact", () => {
|
||||
expect(checkPrimaryTypeMatch("observed_problem", ["observed_problem"])).toBe(true);
|
||||
});
|
||||
|
||||
it("does not match when primary type differs", () => {
|
||||
expect(checkPrimaryTypeMatch("unexplained_change", ["observed_problem"])).toBe(false);
|
||||
});
|
||||
|
||||
it("matches when primary type is in list of expected types", () => {
|
||||
expect(checkPrimaryTypeMatch("observed_problem", ["observed_problem", "fault_report"])).toBe(true);
|
||||
expect(checkPrimaryTypeMatch("unexplained_change", ["observed_problem", "fault_report"])).toBe(false);
|
||||
});
|
||||
|
||||
it("matches reasoning mode when present in list", () => {
|
||||
expect(checkReasoningModeMatch(["establish_baseline", "identify_difference"], ["establish_baseline"])).toBe(true);
|
||||
});
|
||||
|
||||
it("does not match reasoning mode when absent", () => {
|
||||
expect(checkReasoningModeMatch(["validate_claim"], ["establish_baseline"])).toBe(false);
|
||||
});
|
||||
|
||||
it("handles empty lists gracefully", () => {
|
||||
expect(checkPrimaryTypeMatch(null, [])).toBe(false);
|
||||
expect(checkPrimaryTypeMatch("observed_problem", [])).toBe(false);
|
||||
expect(checkReasoningModeMatch([], ["establish_baseline"])).toBe(false);
|
||||
});
|
||||
|
||||
it("normalises whitespace in comparison", () => {
|
||||
expect(normalise("hello world")).toBe("hello world");
|
||||
expect(normalise("Test_With-Symbols!")).toBe("test_with_symbols");
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Paired test case loading
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("paired test cases", () => {
|
||||
const pairedTests = [
|
||||
{ id: "p1a", input: "All customers cannot download invoices.", expectedPrimaryTypes: ["observed_problem"], notes: "Universal scope" },
|
||||
{ id: "p1b", input: "Some customers cannot download invoices.", expectedPrimaryTypes: ["observed_problem"], notes: "Subset scope — key difference from p1a" },
|
||||
{ id: "p2a", input: "Complaints increased by 35%.", expectedPrimaryTypes: ["unexplained_change"], notes: "Isolated metric change" },
|
||||
{ id: "p2b", input: "Complaints increased by 35% while production increased by 40%.", expectedPrimaryTypes: ["unexplained_change"], notes: "Context changes significance" },
|
||||
{ id: "p3a", input: "Sales are falling.", expectedPrimaryTypes: ["observed_problem"], notes: "Vague claim" },
|
||||
{ id: "p3b", input: "Sales fell sharply immediately after the price increase.", expectedPrimaryTypes: ["causal_claim"], notes: "Adds temporal anchor and cause" },
|
||||
{ id: "p4a", input: "I think therefore I am.", expectedPrimaryTypes: ["ambiguous_statement"], notes: "Philosophical statement" },
|
||||
{ id: "p4b", input: "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.", expectedPrimaryTypes: ["question"], notes: "Meta-context changes classification" },
|
||||
];
|
||||
|
||||
it.each(pairedTests)("paired test '%s' loads correctly", (tc) => {
|
||||
expect(tc.id).toBeDefined();
|
||||
expect(tc.input.length).toBeGreaterThan(0);
|
||||
expect(Array.isArray(tc.expectedPrimaryTypes)).toBe(true);
|
||||
expect(tc.notes.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("has meaningful differences between paired test A and B inputs", () => {
|
||||
const p1a = pairedTests.find((t) => t.id === "p1a");
|
||||
const p1b = pairedTests.find((t) => t.id === "p1b");
|
||||
expect(p1a.input).toContain("All customers");
|
||||
expect(p1b.input).toContain("Some customers");
|
||||
});
|
||||
|
||||
it("has at least 8 test cases covering different classification types", () => {
|
||||
const coveredTypes = new Set(pairedTests.map((tc) => tc.expectedPrimaryTypes[0]));
|
||||
expect(coveredTypes.size).toBeGreaterThanOrEqual(4); // at least 4 different types
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Confidence and importance value validation
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("confidence and importance enums", () => {
|
||||
it("has exactly three confidence values: low, medium, high", () => {
|
||||
const validConfidences = ["low", "medium", "high"];
|
||||
for (const c of validConfidences) {
|
||||
expect(confidenceEnum.safeParse(c).success).toBe(true);
|
||||
}
|
||||
// CONFIDENCE_VALUES should match
|
||||
expect(CONFIDENCE_VALUES).toEqual(["low", "medium", "high"]);
|
||||
});
|
||||
|
||||
it("has exactly four importance values", () => {
|
||||
const validImportances = ["incidental", "supporting", "important", "critical"];
|
||||
for (const imp of validImportances) {
|
||||
expect(evidenceRecordSchema.safeParse({ id: "x", description: "y", evidenceType: "direct_observation", confidence: "high", importance: imp }).success).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects values outside the defined enums", () => {
|
||||
expect(confidenceEnum.safeParse("very_high").success).toBe(false);
|
||||
expect(evidenceRecordSchema.safeParse({ id: "x", description: "y", evidenceType: "direct_observation", confidence: "high", importance: "critical_plus" }).success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Missing next question test
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("missing next question handling", () => {
|
||||
it("schema allows optional nextQuestion for ambiguous inputs", () => {
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: { primaryType: "ambiguous_statement", secondaryTypes: [], reasoningModes: [], classificationReason: "Cannot ask meaningful question.", confidence: "low" },
|
||||
reconstruction: { summary: "Ambiguous philosophical statement detected.", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [], nextQuestion: undefined,
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("schema rejects missing required fields", () => {
|
||||
const result = reconstructionV2Schema.safeParse({});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Mock evaluation run test
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("mock evaluation", () => {
|
||||
function normalise(text) {
|
||||
return String(text).toLowerCase().replace(/[^\w\s_]/g, " ").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
it("mock provider can generate deterministic v0.2 output", async () => {
|
||||
// Test that the evaluator's mock provider produces valid schema output
|
||||
const mockInput = "All customers cannot download invoices.";
|
||||
|
||||
// The normaliser should work correctly
|
||||
const normed = normalise(mockInput);
|
||||
expect(normed).toContain("customers");
|
||||
expect(normed).toContain("invoices");
|
||||
});
|
||||
|
||||
it("mock evaluation logic produces expected classification for 'all' vs 'some'", () => {
|
||||
// Verify the evaluator's mock logic handles the key distinction
|
||||
const allInput = "All customers cannot download invoices.";
|
||||
const someInput = "Some customers cannot download invoices.";
|
||||
|
||||
const hasAllWord = /\ball\b|\bno one\b|\bevery\b/i.test(allInput);
|
||||
const hasSomeWord = /some\b/i.test(someInput);
|
||||
|
||||
expect(hasAllWord).toBe(true);
|
||||
expect(hasSomeWord).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
{"id":"tc-001","input":"All customers cannot download their invoices.","expectedPrimaryTypes":["observed_problem","fault_report"],"expectedReasoningModes":["validate_claim","identify_missing_information"],"shouldIdentify":["customers","invoices","download","access_issue"],"shouldNotInfer":[],"notes":"Full scope problem — every customer is affected. Should not infer root cause."}
|
||||
{"id":"tc-002","input":"Some customers cannot download their invoices.","expectedPrimaryTypes":["observed_problem","fault_report"],"expectedReasoningModes":["validate_claim","identify_difference"],"shouldIdentify":["some_customers","invoices","download"],"shouldNotInfer":["root_cause","payment_system_failure"],"notes":"Partial scope — subset of users affected. The word 'some' is the key distinction from tc-001."}
|
||||
{"id":"tc-003","input":"Complaints increased by 35%.","expectedPrimaryTypes":["unexplained_change"],"expectedReasoningModes":["establish_baseline","validate_measurement"],"shouldIdentify":["complaints","increase","35_percent"],"shouldNotInfer":["cause_of_complaints","customer_dissatisfaction_is_worse"],"notes":"Change in isolation — need baseline to understand significance."}
|
||||
{"id":"tc-004","input":"Complaints increased by 35% while production increased by 40%.","expectedPrimaryTypes":["unexplained_change"],"expectedReasoningModes":["identify_difference","validate_measurement"],"shouldIdentify":["complaints_increase","production_increase","relative_rates"],"shouldNotInfer":["production_quality_declined"],"notes":"Paired with tc-003 — the production context changes meaning significantly."}
|
||||
{"id":"tc-005","input":"Sales are falling.","expectedPrimaryTypes":["observed_problem","unexplained_change"],"expectedReasoningModes":["establish_baseline","validate_measurement"],"shouldIdentify":["sales_decline","direction_negative"],"shouldNotInfer":["cause_of_fall","competitor_action"],"notes":"Vague claim — need baseline, timeline, and definition of 'falling'."}
|
||||
{"id":"tc-006","input":"Sales fell sharply immediately after the price increase.","expectedPrimaryTypes":["observed_problem","causal_claim"],"expectedReasoningModes":["investigate_contradiction","test_possible_explanations"],"shouldIdentify":["sales_decline","price_increase","temporal_correlation"],"shouldNotInfer":["price_increase_caused_the_fall"],"notes":"Paired with tc-005 — adds temporal anchor and proposed cause."}
|
||||
{"id":"tc-007","input":"The quarterly revenue exceeded targets but net profit declined by 12%.","expectedPrimaryTypes":["contradiction","unexplained_change"],"expectedReasoningModes":["investigate_contradiction","identify_missing_information"],"shouldIdentify":["revenue_above_target","profit_decline","divergence"],"shouldNotInfer":["cost_overrun_is_the_cause"],"notes":"Apparent contradiction — revenue up but profit down. Missing cost breakdown."}
|
||||
{"id":"tc-008","input":"Revenue from the premium tier dropped while total revenue grew.","expectedPrimaryTypes":["observed_problem","unexplained_change"],"expectedReasoningModes":["decompose_aggregate","identify_difference"],"shouldIdentify":["premium_tier_decline","total_revenue_growth","segment_cannibalization_risk"],"shouldNotInfer":["pricing_change_occurred"],"notes":"Aggregate masking — total growth hides segment decline."}
|
||||
{"id":"tc-009","input":"We need to improve our customer retention rate.","expectedPrimaryTypes":["decision_request","desired_outcome"],"expectedReasoningModes":["decision_support","identify_missing_information"],"shouldIdentify":["retention_improvement_desired","current_state_unknown"],"shouldNotInfer":["retention_rate_is_low","churn_has_increased"],"notes":"Desired outcome without stating the problem. Need to know if retention is actually bad."}
|
||||
{"id":"tc-010","input":"The system latency went from 200ms to 5 seconds on Tuesday.","expectedPrimaryTypes":["unexplained_change","observed_problem"],"expectedReasoningModes":["reconstruct_transition","identify_missing_information"],"shouldIdentify":["latency_baseline_200ms","latency_spike_5s","timestamp_tuesday"],"shouldNotInfer":["database_cause","release_cause"],"notes":"Specific measurement with timing anchor. Should identify transition but not infer cause."}
|
||||
{"id":"tc-011","input":"The new release should fix the login issue.","expectedPrimaryTypes":["decision_request","causal_claim"],"expectedReasoningModes":["validate_claim","investigate_contradiction"],"shouldIdentify":["proposed_solution","login_issue","solution_claim"],"shouldNotInfer":["login_issue_is_real","release_will_work"],"notes":"Proposed solution before problem is fully understood. Assumes the issue and fix are connected."}
|
||||
{"id":"tc-012","input":"I think therefore I am.","expectedPrimaryTypes":["ambiguous_statement","question"],"expectedReasoningModes":["clarify_meaning","identify_missing_information"],"shouldIdentify":["philosophical_statement","insufficient_operational_context"],"shouldNotInfer":["business_problem_exists","actionable_insight_possible"],"notes":"Ambiguous philosophical statement. Should not try to find operational meaning."}
|
||||
{"id":"tc-013","input":"I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.","expectedPrimaryTypes":["question","ambiguous_statement"],"expectedReasoningModes":["clarify_meaning"],"shouldIdentify":["meta_context","testing_hypothesis","self_reference"],"shouldNotInfer":[],"notes":"Paired with tc-12 — the meta-context changes classification entirely."}
|
||||
{"id":"tc-014","input":"The warehouse manager reported that inventory counts don't match the system.","expectedPrimaryTypes":["reported_claim","observed_problem"],"expectedReasoningModes":["validate_claim","investigate_contradiction"],"shouldIdentify":["warehouse_manager_report","inventory_mismatch","system_discrepancy","source_attribution"],"shouldNotInfer":["theft_occurred","software_bug"],"notes":"Reported claim — must distinguish what was said from what it means."}
|
||||
{"id":"tc-015","input":"We've seen a 35% increase in customer complaints.","expectedPrimaryTypes":["unexplained_change"],"expectedReasoningModes":["establish_baseline","validate_measurement"],"shouldIdentify":["complaints_increase","percentage_metric"],"shouldNotInfer":["product_quality_declined","customer_satisfaction_drop"],"notes":"Needs baseline — is this absolute or relative? Over what period?"}
|
||||
{"id":"tc-016","input":"The number of active users increased by 500%, from 4 to 2,001.","expectedPrimaryTypes":["unexplained_change"],"expectedReasoningModes":["validate_measurement","decompose_aggregate"],"shouldIdentify":["active_users_metric","absolute_vs_relative_growth","small_base_problem"],"shouldNotInfer":["product_success"],"notes":"Misleading absolute count where rate matters. Small base inflates percentage."}
|
||||
{"id":"tc-017","input":"User engagement metrics improved but the support ticket backlog grew by 200%.","expectedPrimaryTypes":["contradiction"],"expectedReasoningModes":["investigate_contradiction","identify_difference"],"shouldIdentify":["engagement_improvement","support_backlog_growth","divergent_metrics"],"shouldNotInfer":["users_are_angry","product_quality_is_worse"],"notes":"Two metrics telling opposite stories. Could mean engagement is superficial."}
|
||||
{"id":"tc-018","input":"The manufacturing team needs better quality control.","expectedPrimaryTypes":["decision_request","fault_report"],"expectedReasoningModes":["decision_support","identify_missing_information"],"shouldIdentify":["manufacturing_team","quality_control_desired"],"shouldNotInfer":["quality_is_bad","defect_rate_is_high"],"notes":"Solution proposed without problem specification. What specific quality issue?"}
|
||||
{"id":"tc-019","input":"All users in the EU region are getting a 403 error when trying to access the dashboard.","expectedPrimaryTypes":["observed_problem","fault_report"],"expectedReasoningModes":["validate_claim","identify_difference"],"shouldIdentify":["eu_region","403_error","access_denied","geographic_scope"],"shouldNotInfer":["gdpr_cause","regulatory_change"],"notes":"Geographic subset fault. Should not infer GDPR as cause without evidence."}
|
||||
{"id":"tc-020","input":"Some users in the EU region are getting a 403 error when trying to access the dashboard.","expectedPrimaryTypes":["observed_problem","fault_report"],"expectedReasoningModes":["validate_claim","identify_difference","decompose_aggregate"],"shouldIdentify":["eu_region_subset","403_error","partial_reachability"],"shouldNotInfer":["all_eu_users_affected"],"notes":"Paired with tc-19 — 'some' vs 'all' is the material difference."}
|
||||
{"id":"tc-021","input":"Production output was 1,200 units last month and 1,180 units this month.","expectedPrimaryTypes":["unexplained_change"],"expectedReasoningModes":["validate_measurement","establish_baseline"],"shouldIdentify":["production_output","month_over_month_decline","absolute_difference"],"shouldNotInfer":["efficiency_loss_occurred","equipment_failure"],"notes":"Small absolute change needs context — 1.7% drop might be normal variation."}
|
||||
{"id":"tc-022","input":"The CFO reported that the company's cash position is healthy.","expectedPrimaryTypes":["reported_claim"],"expectedReasoningModes":["validate_claim","identify_missing_information"],"shouldIdentify":["cfo_statement","cash_position_claim","source_attribution_cfo"],"shouldNotInfer":["cash_is_healthy","financial_stability_is_real"],"notes":"Reported opinion — must distinguish what was said from reality."}
|
||||
{"id":"tc-023","input":"We have enough funding to operate for 18 months.","expectedPrimaryTypes":["decision_request","observed_problem"],"expectedReasoningModes":["validate_claim","identify_missing_information"],"shouldIdentify":["funding_period","operational_sustainability","burn_rate_unknown"],"shouldNotInfer":["no_risk_exists"],"notes":"Claim about sustainability without burn rate context."}
|
||||
{"id":"tc-024","input":"The new feature was deployed at 3am and user complaints tripled the next day.","expectedPrimaryTypes":["causal_claim","unexplained_change"],"expectedReasoningModes":["test_possible_explanations","reconstruct_transition"],"shouldIdentify":["feature_deployment","timing_3am","complaint_tripling","temporal_relationship"],"shouldNotInfer":["deployment_caused_complaints"],"notes":"Temporal proximity ≠ causation. Should identify both events but not claim cause."}
|
||||
{"id":"tc-025","input":"We need to launch a mobile app to capture market share.","expectedPrimaryTypes":["decision_request","desired_outcome"],"expectedReasoningModes":["decision_support","identify_missing_information"],"shouldIdentify":["mobile_app_proposed","market_share_desired"],"shouldNotInfer":["no_mobile_app_exists","competitors_have_apps"],"notes":"Desired outcome without problem statement. What evidence supports this decision?"}
|
||||
{"id":"tc-026","input":"The system has been running for 90 days without failure since the migration.","expectedPrimaryTypes":["observed_problem"],"expectedReasoningModes":["validate_claim","establish_baseline"],"shouldIdentify":["uptime_90_days","post_migration_context","baseline_established"],"shouldNotInfer":["system_is_stable_forever"],"notes":"Positive claim about system stability with temporal anchor."}
|
||||
{"id":"tc-027","input":"No one has submitted the required compliance report despite multiple reminders.","expectedPrimaryTypes":["observed_problem","fault_report"],"expectedReasoningModes":["validate_claim","identify_missing_information","investigate_contradiction"],"shouldIdentify":["compliance_report","multiple_reminders","non_submission","absent_action"],"shouldNotInfer":["deliberate_refusal","negligence"],"notes":"Expected-but-missing information. Action was required but absent."}
|
||||
{"id":"tc-028","input":"The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.","expectedPrimaryTypes":["observed_problem","contradiction"],"expectedReasoningModes":["validate_measurement","decompose_aggregate"],"shouldIdentify":["audit_findings","incorrect_reports_rate_3_of_10","data_accuracy_issue"],"shouldNotInfer":["intentional_falsification","systemic_failure"],"notes":"Aggregate data — 30% error rate requires context about severity."}
|
||||
{"id":"tc-029","input":"We should implement the new CRM because our competitors have one.","expectedPrimaryTypes":["decision_request","causal_claim"],"expectedReasoningModes":["test_possible_explanations","validate_claim"],"shouldIdentify":["crm_proposal","competitor_comparison","competitive_pressure"],"shouldNotInfer":["crm_will_help","we_lack_crm","competitors_success_is_from_crm"],"notes":"FOMO-driven decision request without problem analysis."}
|
||||
{"id":"tc-030","input":"The server response time was acceptable last quarter but degraded this month.","expectedPrimaryTypes":["unexplained_change","observed_problem"],"expectedReasoningModes":["reconstruct_transition","identify_missing_information"],"shouldIdentify":["response_time_baseline_acceptable","degradation_timeline","quarter_to_month_comparison"],"shouldNotInfer":["load_increase_occurred"],"notes":"Baseline comparison with transition over time. Need specifics."}
|
||||
{"id":"tc-031","input":"The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.","expectedPrimaryTypes":["contradiction","observed_problem"],"expectedReasoningModes":["validate_claim","investigate_contradiction","identify_missing_information"],"shouldIdentify":["regulatory_requirement","data_location_violation","cross_border_backup"],"shouldNotInfer":["compliance_failure_is_certain"],"notes":"Regulatory conflict — requires verification of both claim and current state."}
|
||||
{"id":"tc-032","input":"External analysts expect our industry to decline by 15% next year due to regulatory changes.","expectedPrimaryTypes":["causal_claim","reported_claim"],"expectedReasoningModes":["validate_claim","test_possible_explanations"],"shouldIdentify":["industry_decline_prediction","external_source","regulatory_cause","15_percent_forecast"],"shouldNotInfer":["decline_will_occur"],"notes":"External prediction — must treat as claim, not fact."}
|
||||
{"id":"tc-033","input":"The database schema was changed on Friday but the reports are still working.","expectedPrimaryTypes":["unexplained_change"],"expectedReasoningModes":["validate_claim","test_possible_explanations"],"shouldIdentify":["schema_change","reports_working_post_change","unexpected_continuity"],"shouldNotInfer":["change_was_harmless"],"notes":"Expected impact did not occur — should flag as unexplained."}
|
||||
{"id":"tc-034","input":"Some team members say the new process is better while others say it's slower.","expectedPrimaryTypes":["contradiction","observed_problem"],"expectedReasoningModes":["validate_claim","investigate_contradiction","identify_missing_information"],"shouldIdentify":["subjective_split","new_process_evaluation","conflicting_opinions","measurement_gap"],"shouldNotInfer":["process_is_better_or_worse"],"notes":"Conflicting subjective claims — need measurable criteria."}
|
||||
{"id":"tc-035","input":"The application works fine on Chrome but not on Safari.","expectedPrimaryTypes":["observed_problem","fault_report"],"expectedReasoningModes":["validate_claim","identify_difference"],"shouldIdentify":["chrome_compatibility","safari_incompatibility","browser_specific_issue"],"shouldNotInfer":["webkit_bug"],"notes":"Browser-specific fault. Should identify the difference but not the technical cause."}
|
||||
Reference in New Issue
Block a user