chore: establish clean v0.2 baseline

Include only the working reconstruction prototype with Ollama integration:
- double-wrapping fix (lib/llm/provider.js)
- explicit v0.2 JSON output schema (prompts/reconstruct-v0.2.md)
- Zod validation layer (lib/reconstruction/schema.js)
- shared core analysis path (lib/analysis.js)
- prompt versioning infrastructure (lib/reconstruction/prompt.js)
- provider abstraction
- functioning Ollama provider path
- updated API route with centralized analysis
- UI components displaying v0.2 data and validation errors
- .gitignore rules for generated evaluation artifacts

Exclude: evaluator experiments, diagnostic tests, debug scripts,
generated artifacts, comparison findings, test data tied to evaluator.
This commit is contained in:
2026-08-01 14:45:06 +01:00
parent a2f9e472ea
commit d72c7c5465
11 changed files with 1048 additions and 172 deletions
+43 -4
View File
@@ -10,17 +10,40 @@ const ValidationIndicator = ({ status }) => {
invalid: "❌ Validation failed",
};
return (
<div className={`flex items-center gap-2 ${styles[status] || "text-gray-500"}`}>
<div
className={`flex items-center gap-2 ${styles[status] || "text-gray-500"}`}
>
<span className="font-medium">{labels[status] || status}</span>
</div>
);
};
const validationIcons = {
valid: "✅",
partial: "⚠️",
invalid: "❌",
};
export default function DiagnosticsView({ result }) {
if (!result) return null;
const metrics = [
{ label: "Model", value: result.modelName || "?" },
{ label: "Duration", value: result.responseDurationMs != null ? `${result.responseDurationMs}ms` : "?" },
{ label: "Validation", value: <ValidationIndicator status={result.validationStatus || "invalid"} /> },
{ label: "Provider", value: "Ollama" },
{ label: "Prompt version", value: result.promptVersion || "?" },
{
label: "Duration",
value:
result.responseDurationMs != null
? `${result.responseDurationMs}ms`
: "?",
},
{
label: "Validation",
value: (
<ValidationIndicator status={result.validationStatus || "invalid"} />
),
},
];
return (
@@ -35,16 +58,32 @@ export default function DiagnosticsView({ result }) {
))}
</dl>
{/* Collapsed raw output for debugging */}
{result.rawResponse && (
<details className="mt-4">
<summary className="cursor-pointer text-xs text-gray-500 underline hover:text-gray-700">
View raw model response
View raw model response (
{(result.rawResponse?.length || 0).toLocaleString()} chars)
</summary>
<pre className="mt-2 max-h-60 overflow-auto rounded bg-gray-900 px-3 py-2 text-xs leading-relaxed text-green-400">
{result.rawResponse}
</pre>
</details>
)}
{/* Errors if present */}
{result.errors && result.errors.length > 0 && (
<details className="mt-3">
<summary className="cursor-pointer text-xs text-red-500 underline hover:text-red-700">
Validation errors ({result.errors.length})
</summary>
<ul className="mt-1 space-y-0.5 text-xs text-red-600">
{result.errors.map((err, i) => (
<li key={i}>{err}</li>
))}
</ul>
</details>
)}
</div>
);
}
+382 -42
View File
@@ -1,15 +1,8 @@
const categoryLabels = {
observations: "Direct Observations",
reportedClaims: "Reported Claims",
assumptions: "Unsupported Assumptions",
entities: "Entities",
transitions: "Transitions",
expectedButMissing: "Expected But Missing",
presentButUnexpected: "Present But Unexpected",
contradictions: "Contradictions",
openUncertainties: "Open Uncertainties",
};
"use client";
import { useMemo } from "react";
// ── Confidence badge (shared) ────────────────────────
const confidenceColor = {
low: "text-red-600 bg-red-50 border-red-200",
medium: "text-yellow-700 bg-yellow-50 border-yellow-200",
@@ -17,54 +10,401 @@ const confidenceColor = {
};
const ConfidenceBadge = ({ level }) => (
<span className={`inline-block rounded-full border px-2 py-0.5 text-xs font-medium ${confidenceColor[level] || "text-gray-600 bg-gray-100"}`}>
<span
className={`inline-block rounded-full border px-2 py-0.5 text-xs font-medium ${confidenceColor[level] || "text-gray-600 bg-gray-100"}`}
>
{level}
</span>
);
function ItemList({ items, renderExtra }) {
if (!items?.length) return <p className="text-sm italic text-gray-400">None identified</p>;
// ── Evidence type labels (shared) ───────────────────
const evidenceTypeLabels = {
direct_observation: "Direct Observation",
reported_statement: "Reported Statement",
interpretation: "Interpretation",
assumption: "Assumption",
inferred_relationship: "Inferred Relationship",
};
const importanceColors = {
incidental: "text-gray-500 bg-gray-50 border-gray-200",
supporting: "text-blue-700 bg-blue-50 border-blue-200",
important: "text-orange-700 bg-orange-50 border-orange-200",
critical: "text-red-800 bg-red-50 border-red-300 font-semibold",
};
const importanceLabels = {
incidental: "Incidental",
supporting: "Supporting",
important: "Important",
critical: "Critical",
};
// ── Input classification display ────────────────────
function ClassificationDisplay({ classification }) {
if (!classification) return null;
const p = classification.primaryType || classification.primary_type;
const sec =
classification.secondaryTypes || classification.secondary_types || [];
const modes =
classification.reasoningModes || classification.reasoning_modes || [];
// Normalize camelCase to snake_case for display if needed
const primaryLabel = String(p)
.replace(/_/g, " ")
.replace(/\b\w/g, (c) => c.toUpperCase());
const secLabels = sec.map((s) =>
s.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()),
);
const modeLabels = modes.map((m) =>
m.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()),
);
return (
<ul className="space-y-2">
{items.map((item) => (
<li key={item.id} className="rounded border border-gray-200 bg-white px-3 py-2 text-sm">
<div className="flex items-center gap-2">
<span className="font-mono text-xs text-gray-400">#{item.id}</span>
<ConfidenceBadge level={item.confidence} />
</div>
<p className="mt-1">{item.description}</p>
{renderExtra && renderExtra(item)}
</li>
))}
</ul>
<div className="rounded-lg border border-blue-200 bg-blue-50 p-4">
<h3 className="mb-2 text-sm font-semibold text-blue-700">
Input Classification
</h3>
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1.5 text-sm">
<dt className="text-blue-500">Primary type</dt>
<dd className="font-medium">{primaryLabel}</dd>
{secLabels.length > 0 && (
<>
<dt className="text-blue-500 pt-1">Secondary types</dt>
<dd>{secLabels.join(" · ")}</dd>
</>
)}
{modeLabels.length > 0 && (
<>
<dt className="text-blue-500 pt-1">Reasoning modes</dt>
<dd>{modeLabels.join(" · ")}</dd>
</>
)}
<dt className="text-blue-500 pt-1">Classification reason</dt>
<dd className="italic">
{classification.classificationReason ||
classification.classification_reason}
</dd>
<dt className="text-blue-500 pt-1">Confidence</dt>
<dd>
<ConfidenceBadge level={classification.confidence} />
</dd>
</dl>
</div>
);
}
// ── Reconstruction summary ──────────────────────────
function SummaryDisplay({ reconstruction }) {
if (!reconstruction?.summary) return null;
const summary = reconstruction.summary || reconstruction.Summary;
return (
<div className="rounded-lg border border-gray-200 bg-white p-4">
<h3 className="mb-2 text-sm font-semibold text-gray-600">
Reconstruction Summary
</h3>
<p className="text-sm leading-relaxed">{summary}</p>
</div>
);
}
// ── Generic item list (used for multiple sections) ──
function ItemList({ title, items, renderExtra }) {
const count = items?.length;
if (!count) return null; // hide empty sections entirely
const itemsArr = Array.isArray(items) ? items : [items];
return (
<div className="mb-4 rounded-lg border border-gray-200 bg-white p-4">
<h3 className="mb-2 text-sm font-semibold text-gray-600">
{title} ({count})
</h3>
<ul className="space-y-2">
{itemsArr.map((item, idx) => (
<li
key={item.id || `${title}-${idx}`}
className="rounded border border-gray-200 bg-white px-3 py-2 text-sm"
>
<div className="flex items-center gap-2">
{item.id && (
<span className="font-mono text-xs text-gray-400">
#{item.id}
</span>
)}
{item.confidence && <ConfidenceBadge level={item.confidence} />}
{item.importance && (
<span
className={`inline-block rounded-full border px-2 py-0.5 text-xs font-medium ${importanceColors[item.importance] || "text-gray-600 bg-gray-100"}`}
>
{importanceLabels[item.importance]}
</span>
)}
</div>
<p className="mt-1">{item.description}</p>
{renderExtra && renderExtra(item)}
</li>
))}
</ul>
</div>
);
}
// ── Plausible interpretations ───────────────────────
function InterpretationsDisplay({ interpretations }) {
if (!interpretations?.length) return null;
const arr = Array.isArray(interpretations)
? interpretations
: [interpretations];
return (
<div className="mb-4 rounded-lg border border-indigo-200 bg-indigo-50 p-4">
<h3 className="mb-2 text-sm font-semibold text-indigo-700">
Plausible Interpretations ({arr.length})
</h3>
<ul className="space-y-3">
{arr.map((interp, idx) => (
<li
key={interp.id || `${idx}`}
className="rounded border border-indigo-200 bg-white px-3 py-2.5 text-sm leading-relaxed"
>
<div className="flex items-center gap-2 mb-1">
<span className="font-medium text-indigo-600">
{interp.description}
</span>
{interp.confidence && (
<ConfidenceBadge level={interp.confidence} />
)}
</div>
{interp.supportingEvidenceIds?.length > 0 && (
<p className="text-xs text-gray-500">
Supporting evidence: {interp.supportingEvidenceIds.join(", ")}
</p>
)}
{interp.assumptionsRequired?.length > 0 && (
<p className="text-xs italic text-gray-500">
Requires assumptions: {interp.assumptionsRequired.join("; ")}
</p>
)}
</li>
))}
</ul>
</div>
);
}
// ── Next question (prominent) ───────────────────────
function NextQuestionDisplay({ question }) {
if (!question?.question) return null;
const q = question.question || question.Question;
const targets = question.targets || question.Targets || [];
const reason = question.reason || question.Reason || "";
const value =
question.expectedInformationValue ||
question.expected_information_value ||
"medium";
const valueLabel =
{ low: "Low", medium: "Medium", high: "High" }[value] || "Medium";
const valueColor =
{
low: "bg-yellow-100 text-yellow-800",
medium: "bg-blue-100 text-blue-800",
high: "bg-green-100 text-green-800",
}[value] || "";
return (
<div className="rounded-lg border-2 border-green-300 bg-green-50 p-5">
<div className="flex items-center gap-2 mb-2">
<h3 className="text-sm font-bold text-green-800">Next Question</h3>
<span
className={`rounded-full px-2 py-0.5 text-xs font-medium ${valueColor}`}
>
{valueLabel} value
</span>
</div>
<p className="mb-2 text-base font-medium text-gray-900">{q}</p>
{targets.length > 0 && (
<p className="text-sm text-gray-600">Targets: {targets.join(", ")}</p>
)}
{reason && (
<p className="text-sm italic text-gray-500">Because: {reason}</p>
)}
</div>
);
}
// ── Evidence list ───────────────────────────────────
function EvidenceDisplay({ evidence }) {
if (!evidence?.length) return null;
const arr = Array.isArray(evidence) ? evidence : [evidence];
const evidenceLabels = {
direct_observation: "👁 Direct Observation",
reported_statement: "🗣 Reported Statement",
interpretation: "💡 Interpretation",
assumption: "❓ Assumption",
inferred_relationship: "🔗 Inferred Relationship",
};
return (
<div className="mb-4 rounded-lg border border-gray-200 bg-white p-4">
<h3 className="mb-2 text-sm font-semibold text-gray-600">
Supporting Evidence ({arr.length})
</h3>
<ul className="space-y-2">
{arr.map((item, idx) => (
<li
key={item.id || `${idx}`}
className="rounded border border-gray-200 bg-white px-3 py-2 text-sm leading-relaxed"
>
<div className="flex items-center gap-2 mb-0.5 flex-wrap">
{item.id && (
<span className="font-mono text-xs text-gray-400">
#{item.id}
</span>
)}
<span
className={`inline-block rounded px-1.5 py-0.5 text-[10px] font-medium ${importanceColors[item.importance] || "text-gray-600 bg-gray-100"}`}
>
{importanceLabels[item.importance]}
</span>
<span className="inline-block rounded px-1.5 py-0.5 text-[10px] font-medium bg-gray-100 text-gray-700">
{evidenceLabels[item.evidenceType] || item.evidenceType}
</span>
{item.confidence && <ConfidenceBadge level={item.confidence} />}
</div>
<p className="text-sm">{item.description}</p>
{(item.source || item.attribution) && (
<p className="mt-0.5 text-xs text-gray-400">
Source: {item.source || item.attribution}
</p>
)}
</li>
))}
</ul>
</div>
);
}
// ── Main component ──────────────────────────────────
export default function ReconstructionView({ reconstruction, partial }) {
// Handle both v0.2 direct object and wrapped result formats
const data = reconstruction;
if (partial) {
return (
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-3 text-sm text-yellow-800">
Partial result some fields failed validation. Showing what was accepted.
Partial result some fields failed validation. Showing what was
accepted.
</div>
);
}
const categories = Object.entries(categoryLabels).map(([key, label]) => ({
key,
label,
items: reconstruction[key],
}));
return (
<div className="space-y-1">
<h2 className="mb-3 text-lg font-semibold">Reconstruction</h2>
{categories.map(({ key, label, items }) => (
<div key={key} className="mb-4 rounded border border-gray-200 bg-white p-4">
<h3 className="mb-2 text-sm font-medium text-gray-600">{label}</h3>
<ItemList items={items} />
</div>
))}
<div className="space-y-4">
{/* Classification first */}
{data.inputClassification && (
<ClassificationDisplay classification={data.inputClassification} />
)}
{/* Summary */}
{data.reconstruction?.summary && (
<SummaryDisplay reconstruction={data.reconstruction} />
)}
{/* Key differences */}
{data.reconstruction?.differences && (
<ItemList
title="Key Differences"
items={data.reconstruction.differences}
/>
)}
{/* Unexplained transitions */}
{data.reconstruction?.unexplainedTransitions &&
data.reconstruction.unexplainedTransitions.length > 0 && (
<ItemList
title="Unexplained Transitions"
items={data.reconstruction.unexplainedTransitions}
renderExtra={(i) =>
i.entity && (
<p className="mt-1 text-xs text-gray-500">Entity: {i.entity}</p>
)
}
/>
)}
{/* Contradictions */}
{data.reconstruction?.contradictions &&
data.reconstruction.contradictions.length > 0 && (
<ItemList
title="Contradictions"
items={data.reconstruction.contradictions}
/>
)}
{/* Important unknowns */}
{data.reconstruction?.importantUnknowns &&
data.reconstruction.importantUnknowns.length > 0 && (
<ItemList
title="Important Unknowns"
items={data.reconstruction.importantUnknowns}
/>
)}
{/* Plausible interpretations */}
{data.reconstruction?.plausibleInterpretations &&
data.reconstruction.plausibleInterpretations.length > 0 && (
<InterpretationsDisplay
interpretations={data.reconstruction.plausibleInterpretations}
/>
)}
{/* Secondary reconstruction categories (actors, systems, etc.) */}
{data.reconstruction?.actors && data.reconstruction.actors.length > 0 && (
<ItemList title="Actors" items={data.reconstruction.actors} />
)}
{data.reconstruction?.systemsOrObjects &&
data.reconstruction.systemsOrObjects.length > 0 && (
<ItemList
title="Systems / Objects"
items={data.reconstruction.systemsOrObjects}
/>
)}
{data.reconstruction?.expectedStates &&
data.reconstruction.expectedStates.length > 0 && (
<ItemList
title="Expected States"
items={data.reconstruction.expectedStates}
/>
)}
{data.reconstruction?.observedStates &&
data.reconstruction.observedStates.length > 0 && (
<ItemList
title="Observed States"
items={data.reconstruction.observedStates}
/>
)}
{data.reconstruction?.knownTransitions &&
data.reconstruction.knownTransitions.length > 0 && (
<ItemList
title="Known Transitions"
items={data.reconstruction.knownTransitions}
renderExtra={(i) => (
<div className="mt-1 text-xs text-gray-500">
{i.entity && <span>Entity: {i.entity} · </span>}
From &ldquo;{i.previousState}&rdquo; To &ldquo;{i.currentState}&rdquo; (&quot;{i.explanationStatus}&quot;)
</div>
)}
/>
)}
{/* Next question — prominent */}
<NextQuestionDisplay question={data.nextQuestion} />
{/* Evidence */}
{data.evidence && <EvidenceDisplay evidence={data.evidence} />}
</div>
);
}
+50 -22
View File
@@ -8,7 +8,7 @@ const MAX_LENGTH = 10000;
export default function ScenarioForm() {
const [scenario, setScenario] = useState("");
const [status, setStatus] = useState("idle"); // idle | loading | error | success
const [status, setStatus] = useState("idle"); // idle | loading | error | success | partial
const [result, setResult] = useState(null);
const textareaRef = useRef(null);
@@ -29,6 +29,10 @@ export default function ScenarioForm() {
if (res.ok && data.validationStatus === "valid") {
setStatus("success");
setResult(data);
} else if (data.success) {
// Success in analysis but validation may be partial
setStatus("success");
setResult(data);
} else {
setStatus("error");
setResult(data);
@@ -39,8 +43,13 @@ export default function ScenarioForm() {
}
};
// Always show diagnostics when there's a result (even if validation failed)
const hasDiagnostics = result && (result.reconstruction || result.modelName || result.responseDurationMs !== undefined);
// Determine if we have meaningful content to display
const hasClassification = result?.inputClassification;
const hasReconstruction = result?.reconstruction;
const hasNextQuestion = result?.nextQuestion;
const hasEvidence = result?.evidence && result.evidence.length > 0;
const hasMeaningfulContent =
hasClassification || hasReconstruction || hasNextQuestion || hasEvidence;
return (
<div className="space-y-6">
@@ -54,7 +63,9 @@ export default function ScenarioForm() {
className="w-full rounded-lg border border-gray-300 px-4 py-3 text-sm focus:border-gray-500 focus:outline-none focus:ring-2 focus:ring-gray-400"
/>
<div className="flex items-center justify-between">
<span className="text-xs text-gray-400">{scenario.length}/{MAX_LENGTH}</span>
<span className="text-xs text-gray-400">
{scenario.length}/{MAX_LENGTH}
</span>
<button
type="submit"
disabled={status === "loading" || !scenario.trim()}
@@ -65,6 +76,7 @@ export default function ScenarioForm() {
</div>
</form>
{/* Error state */}
{status === "error" && (
<div className="space-y-3">
{result?.error && (
@@ -72,35 +84,51 @@ export default function ScenarioForm() {
Error: {result.error}
</div>
)}
{hasDiagnostics && result?.modelName && (
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1.5 text-sm">
<dt className="text-gray-500">Model</dt>
<dd>{result.modelName}</dd>
<dt className="text-gray-500">Duration</dt>
<dd>{result.responseDurationMs != null ? `${result.responseDurationMs}ms` : "?"}</dd>
</dl>
{/* Show partial content even on validation failure */}
{(hasClassification || hasReconstruction) && (
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
Partial result some fields failed validation. Showing what was
accepted.
</div>
)}
{hasReconstruction && (
<ReconstructionView reconstruction={result} partial />
)}
</div>
)}
{status === "success" && result?.reconstruction && (
{/* Success state */}
{status === "success" && hasMeaningfulContent && (
<div className="space-y-4">
<ReconstructionView reconstruction={result.reconstruction} />
<DiagnosticsView result={result} />
<ReconstructionView reconstruction={result} />
</div>
)}
{status === "error" && result?.reconstruction && (
<div className="space-y-3">
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
Partial result some fields failed validation. Showing what was accepted.
</div>
<ReconstructionView reconstruction={result.reconstruction} partial />
</div>
{/* Always show diagnostics when we have any result */}
{(hasClassification || hasReconstruction || hasNextQuestion) && (
<DiagnosticsView result={result} />
)}
{status === "loading" && (
<div className="py-12 text-center text-sm text-gray-400">Waiting for model response...</div>
<div className="py-12 text-center text-sm text-gray-400">
Waiting for model response...
</div>
)}
{/* Empty state */}
{status === "idle" && (
<div className="rounded-lg border border-dashed border-gray-300 bg-gray-50 px-6 py-8 text-center">
<p className="text-sm text-gray-400">
Enter a scenario above and click Analyse to begin.
</p>
</div>
)}
{/* Invalid result with no partial data */}
{status === "error" && !result?.error && !hasMeaningfulContent && (
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
Validation failed no structured output was produced.
</div>
)}
</div>
);