chore: establish clean v0.2 baseline

Include only the working reconstruction prototype with Ollama integration:
- double-wrapping fix (lib/llm/provider.js)
- explicit v0.2 JSON output schema (prompts/reconstruct-v0.2.md)
- Zod validation layer (lib/reconstruction/schema.js)
- shared core analysis path (lib/analysis.js)
- prompt versioning infrastructure (lib/reconstruction/prompt.js)
- provider abstraction
- functioning Ollama provider path
- updated API route with centralized analysis
- UI components displaying v0.2 data and validation errors
- .gitignore rules for generated evaluation artifacts

Exclude: evaluator experiments, diagnostic tests, debug scripts,
generated artifacts, comparison findings, test data tied to evaluator.
This commit is contained in:
2026-08-01 14:45:06 +01:00
parent a2f9e472ea
commit d72c7c5465
11 changed files with 1048 additions and 172 deletions
+5
View File
@@ -34,3 +34,8 @@ Thumbs.db
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# Generated evaluation artifacts (regenerated each run)
evaluation-results/
provider-debug-results/
tests-results/
+26 -78
View File
@@ -1,101 +1,49 @@
import { getConfig } from "@/lib/config";
import { getProvider } from "@/lib/llm/provider";
import { reconstructionSchema } from "@/lib/reconstruction/schema";
const MAX_SCENARIO_LENGTH = 10000;
import {
analyseScenario,
PROMPT_VERSIONS,
DEFAULT_PROMPT_VERSION,
} from "@/lib/analysis";
export async function POST(request) {
const startTime = Date.now();
let rawResponse = null;
try {
const body = await request.json();
if (!body.scenario || typeof body.scenario !== "string") {
return Response.json(
{ error: "Request must include a 'scenario' string field" },
{ status: 400 }
{ status: 400 },
);
}
const trimmed = body.scenario.trim();
// Optional prompt version override
let promptVersion = DEFAULT_PROMPT_VERSION;
if (body.promptVersion && PROMPT_VERSIONS.includes(body.promptVersion)) {
promptVersion = body.promptVersion;
}
if (trimmed.length === 0) {
const result = await analyseScenario(body.scenario, { promptVersion });
if (!result.success) {
return Response.json(
{ error: "Scenario cannot be empty" },
{ status: 400 }
{ ...result, reconstruction: result.reconstruction || null },
{ status: Number(result.statusCode) || 500 },
);
}
if (trimmed.length > MAX_SCENARIO_LENGTH) {
return Response.json(
{ error: `Scenario must be under ${MAX_SCENARIO_LENGTH} characters` },
{ status: 400 }
);
}
const configResult = getConfig();
if (!configResult.ok) {
return Response.json(
{ error: "Invalid server configuration" },
{ status: 500 }
);
}
const { OLLAMA_BASE_URL, OLLAMA_MODEL } = configResult.config;
const provider = getProvider();
// Attempt parse to capture raw for debugging
let reconstruction;
try {
reconstruction = await provider.generateReconstruction(trimmed, OLLAMA_MODEL);
} catch (e) {
return Response.json(
{
error: e.message || "Unknown server error",
responseDurationMs: Date.now() - startTime,
modelName: OLLAMA_MODEL,
validationStatus: "invalid",
},
{ status: 500 }
);
}
// Try to stringify for rawResponse display (safe even if it's already an object)
try {
rawResponse = JSON.stringify(reconstruction);
} catch {
rawResponse = String(reconstruction).slice(0, 2000);
}
const duration = Date.now() - startTime;
// Validate with Zod schema
const validationResult = reconstructionSchema.safeParse(reconstruction);
if (!validationResult.success) {
return Response.json({
reconstruction: null,
modelName: OLLAMA_MODEL,
responseDurationMs: duration,
validationStatus: "invalid",
rawResponse: rawResponse?.slice(0, 2000),
errors: validationResult.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`),
});
}
return Response.json({
reconstruction: validationResult.data,
modelName: OLLAMA_MODEL,
responseDurationMs: duration,
validationStatus: "valid",
rawResponse: rawResponse?.slice(0, 2000),
inputClassification: result.inputClassification,
reconstruction: result.reconstruction,
evidence: result.evidence,
nextQuestion: result.nextQuestion,
modelName: result.modelName,
responseDurationMs: result.responseDurationMs,
validationStatus: result.validationStatus,
promptVersion: result.promptVersion,
});
} catch (e) {
const duration = Date.now() - startTime;
return Response.json(
{ error: e.message || "Unknown server error", responseDurationMs: duration },
{ status: 500 }
{ error: e.message || "Unknown server error", responseDurationMs: 0 },
{ status: 500 },
);
}
}
+43 -4
View File
@@ -10,17 +10,40 @@ const ValidationIndicator = ({ status }) => {
invalid: "❌ Validation failed",
};
return (
<div className={`flex items-center gap-2 ${styles[status] || "text-gray-500"}`}>
<div
className={`flex items-center gap-2 ${styles[status] || "text-gray-500"}`}
>
<span className="font-medium">{labels[status] || status}</span>
</div>
);
};
const validationIcons = {
valid: "✅",
partial: "⚠️",
invalid: "❌",
};
export default function DiagnosticsView({ result }) {
if (!result) return null;
const metrics = [
{ label: "Model", value: result.modelName || "?" },
{ label: "Duration", value: result.responseDurationMs != null ? `${result.responseDurationMs}ms` : "?" },
{ label: "Validation", value: <ValidationIndicator status={result.validationStatus || "invalid"} /> },
{ label: "Provider", value: "Ollama" },
{ label: "Prompt version", value: result.promptVersion || "?" },
{
label: "Duration",
value:
result.responseDurationMs != null
? `${result.responseDurationMs}ms`
: "?",
},
{
label: "Validation",
value: (
<ValidationIndicator status={result.validationStatus || "invalid"} />
),
},
];
return (
@@ -35,16 +58,32 @@ export default function DiagnosticsView({ result }) {
))}
</dl>
{/* Collapsed raw output for debugging */}
{result.rawResponse && (
<details className="mt-4">
<summary className="cursor-pointer text-xs text-gray-500 underline hover:text-gray-700">
View raw model response
View raw model response (
{(result.rawResponse?.length || 0).toLocaleString()} chars)
</summary>
<pre className="mt-2 max-h-60 overflow-auto rounded bg-gray-900 px-3 py-2 text-xs leading-relaxed text-green-400">
{result.rawResponse}
</pre>
</details>
)}
{/* Errors if present */}
{result.errors && result.errors.length > 0 && (
<details className="mt-3">
<summary className="cursor-pointer text-xs text-red-500 underline hover:text-red-700">
Validation errors ({result.errors.length})
</summary>
<ul className="mt-1 space-y-0.5 text-xs text-red-600">
{result.errors.map((err, i) => (
<li key={i}>{err}</li>
))}
</ul>
</details>
)}
</div>
);
}
+381 -41
View File
@@ -1,15 +1,8 @@
const categoryLabels = {
observations: "Direct Observations",
reportedClaims: "Reported Claims",
assumptions: "Unsupported Assumptions",
entities: "Entities",
transitions: "Transitions",
expectedButMissing: "Expected But Missing",
presentButUnexpected: "Present But Unexpected",
contradictions: "Contradictions",
openUncertainties: "Open Uncertainties",
};
"use client";
import { useMemo } from "react";
// ── Confidence badge (shared) ────────────────────────
const confidenceColor = {
low: "text-red-600 bg-red-50 border-red-200",
medium: "text-yellow-700 bg-yellow-50 border-yellow-200",
@@ -17,54 +10,401 @@ const confidenceColor = {
};
const ConfidenceBadge = ({ level }) => (
<span className={`inline-block rounded-full border px-2 py-0.5 text-xs font-medium ${confidenceColor[level] || "text-gray-600 bg-gray-100"}`}>
<span
className={`inline-block rounded-full border px-2 py-0.5 text-xs font-medium ${confidenceColor[level] || "text-gray-600 bg-gray-100"}`}
>
{level}
</span>
);
function ItemList({ items, renderExtra }) {
if (!items?.length) return <p className="text-sm italic text-gray-400">None identified</p>;
// ── Evidence type labels (shared) ───────────────────
const evidenceTypeLabels = {
direct_observation: "Direct Observation",
reported_statement: "Reported Statement",
interpretation: "Interpretation",
assumption: "Assumption",
inferred_relationship: "Inferred Relationship",
};
const importanceColors = {
incidental: "text-gray-500 bg-gray-50 border-gray-200",
supporting: "text-blue-700 bg-blue-50 border-blue-200",
important: "text-orange-700 bg-orange-50 border-orange-200",
critical: "text-red-800 bg-red-50 border-red-300 font-semibold",
};
const importanceLabels = {
incidental: "Incidental",
supporting: "Supporting",
important: "Important",
critical: "Critical",
};
// ── Input classification display ────────────────────
function ClassificationDisplay({ classification }) {
if (!classification) return null;
const p = classification.primaryType || classification.primary_type;
const sec =
classification.secondaryTypes || classification.secondary_types || [];
const modes =
classification.reasoningModes || classification.reasoning_modes || [];
// Normalize camelCase to snake_case for display if needed
const primaryLabel = String(p)
.replace(/_/g, " ")
.replace(/\b\w/g, (c) => c.toUpperCase());
const secLabels = sec.map((s) =>
s.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()),
);
const modeLabels = modes.map((m) =>
m.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()),
);
return (
<ul className="space-y-2">
{items.map((item) => (
<li key={item.id} className="rounded border border-gray-200 bg-white px-3 py-2 text-sm">
<div className="flex items-center gap-2">
<span className="font-mono text-xs text-gray-400">#{item.id}</span>
<ConfidenceBadge level={item.confidence} />
</div>
<p className="mt-1">{item.description}</p>
{renderExtra && renderExtra(item)}
</li>
))}
</ul>
<div className="rounded-lg border border-blue-200 bg-blue-50 p-4">
<h3 className="mb-2 text-sm font-semibold text-blue-700">
Input Classification
</h3>
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1.5 text-sm">
<dt className="text-blue-500">Primary type</dt>
<dd className="font-medium">{primaryLabel}</dd>
{secLabels.length > 0 && (
<>
<dt className="text-blue-500 pt-1">Secondary types</dt>
<dd>{secLabels.join(" · ")}</dd>
</>
)}
{modeLabels.length > 0 && (
<>
<dt className="text-blue-500 pt-1">Reasoning modes</dt>
<dd>{modeLabels.join(" · ")}</dd>
</>
)}
<dt className="text-blue-500 pt-1">Classification reason</dt>
<dd className="italic">
{classification.classificationReason ||
classification.classification_reason}
</dd>
<dt className="text-blue-500 pt-1">Confidence</dt>
<dd>
<ConfidenceBadge level={classification.confidence} />
</dd>
</dl>
</div>
);
}
// ── Reconstruction summary ──────────────────────────
function SummaryDisplay({ reconstruction }) {
if (!reconstruction?.summary) return null;
const summary = reconstruction.summary || reconstruction.Summary;
return (
<div className="rounded-lg border border-gray-200 bg-white p-4">
<h3 className="mb-2 text-sm font-semibold text-gray-600">
Reconstruction Summary
</h3>
<p className="text-sm leading-relaxed">{summary}</p>
</div>
);
}
// ── Generic item list (used for multiple sections) ──
function ItemList({ title, items, renderExtra }) {
const count = items?.length;
if (!count) return null; // hide empty sections entirely
const itemsArr = Array.isArray(items) ? items : [items];
return (
<div className="mb-4 rounded-lg border border-gray-200 bg-white p-4">
<h3 className="mb-2 text-sm font-semibold text-gray-600">
{title} ({count})
</h3>
<ul className="space-y-2">
{itemsArr.map((item, idx) => (
<li
key={item.id || `${title}-${idx}`}
className="rounded border border-gray-200 bg-white px-3 py-2 text-sm"
>
<div className="flex items-center gap-2">
{item.id && (
<span className="font-mono text-xs text-gray-400">
#{item.id}
</span>
)}
{item.confidence && <ConfidenceBadge level={item.confidence} />}
{item.importance && (
<span
className={`inline-block rounded-full border px-2 py-0.5 text-xs font-medium ${importanceColors[item.importance] || "text-gray-600 bg-gray-100"}`}
>
{importanceLabels[item.importance]}
</span>
)}
</div>
<p className="mt-1">{item.description}</p>
{renderExtra && renderExtra(item)}
</li>
))}
</ul>
</div>
);
}
// ── Plausible interpretations ───────────────────────
function InterpretationsDisplay({ interpretations }) {
if (!interpretations?.length) return null;
const arr = Array.isArray(interpretations)
? interpretations
: [interpretations];
return (
<div className="mb-4 rounded-lg border border-indigo-200 bg-indigo-50 p-4">
<h3 className="mb-2 text-sm font-semibold text-indigo-700">
Plausible Interpretations ({arr.length})
</h3>
<ul className="space-y-3">
{arr.map((interp, idx) => (
<li
key={interp.id || `${idx}`}
className="rounded border border-indigo-200 bg-white px-3 py-2.5 text-sm leading-relaxed"
>
<div className="flex items-center gap-2 mb-1">
<span className="font-medium text-indigo-600">
{interp.description}
</span>
{interp.confidence && (
<ConfidenceBadge level={interp.confidence} />
)}
</div>
{interp.supportingEvidenceIds?.length > 0 && (
<p className="text-xs text-gray-500">
Supporting evidence: {interp.supportingEvidenceIds.join(", ")}
</p>
)}
{interp.assumptionsRequired?.length > 0 && (
<p className="text-xs italic text-gray-500">
Requires assumptions: {interp.assumptionsRequired.join("; ")}
</p>
)}
</li>
))}
</ul>
</div>
);
}
// ── Next question (prominent) ───────────────────────
function NextQuestionDisplay({ question }) {
if (!question?.question) return null;
const q = question.question || question.Question;
const targets = question.targets || question.Targets || [];
const reason = question.reason || question.Reason || "";
const value =
question.expectedInformationValue ||
question.expected_information_value ||
"medium";
const valueLabel =
{ low: "Low", medium: "Medium", high: "High" }[value] || "Medium";
const valueColor =
{
low: "bg-yellow-100 text-yellow-800",
medium: "bg-blue-100 text-blue-800",
high: "bg-green-100 text-green-800",
}[value] || "";
return (
<div className="rounded-lg border-2 border-green-300 bg-green-50 p-5">
<div className="flex items-center gap-2 mb-2">
<h3 className="text-sm font-bold text-green-800">Next Question</h3>
<span
className={`rounded-full px-2 py-0.5 text-xs font-medium ${valueColor}`}
>
{valueLabel} value
</span>
</div>
<p className="mb-2 text-base font-medium text-gray-900">{q}</p>
{targets.length > 0 && (
<p className="text-sm text-gray-600">Targets: {targets.join(", ")}</p>
)}
{reason && (
<p className="text-sm italic text-gray-500">Because: {reason}</p>
)}
</div>
);
}
// ── Evidence list ───────────────────────────────────
function EvidenceDisplay({ evidence }) {
if (!evidence?.length) return null;
const arr = Array.isArray(evidence) ? evidence : [evidence];
const evidenceLabels = {
direct_observation: "👁 Direct Observation",
reported_statement: "🗣 Reported Statement",
interpretation: "💡 Interpretation",
assumption: "❓ Assumption",
inferred_relationship: "🔗 Inferred Relationship",
};
return (
<div className="mb-4 rounded-lg border border-gray-200 bg-white p-4">
<h3 className="mb-2 text-sm font-semibold text-gray-600">
Supporting Evidence ({arr.length})
</h3>
<ul className="space-y-2">
{arr.map((item, idx) => (
<li
key={item.id || `${idx}`}
className="rounded border border-gray-200 bg-white px-3 py-2 text-sm leading-relaxed"
>
<div className="flex items-center gap-2 mb-0.5 flex-wrap">
{item.id && (
<span className="font-mono text-xs text-gray-400">
#{item.id}
</span>
)}
<span
className={`inline-block rounded px-1.5 py-0.5 text-[10px] font-medium ${importanceColors[item.importance] || "text-gray-600 bg-gray-100"}`}
>
{importanceLabels[item.importance]}
</span>
<span className="inline-block rounded px-1.5 py-0.5 text-[10px] font-medium bg-gray-100 text-gray-700">
{evidenceLabels[item.evidenceType] || item.evidenceType}
</span>
{item.confidence && <ConfidenceBadge level={item.confidence} />}
</div>
<p className="text-sm">{item.description}</p>
{(item.source || item.attribution) && (
<p className="mt-0.5 text-xs text-gray-400">
Source: {item.source || item.attribution}
</p>
)}
</li>
))}
</ul>
</div>
);
}
// ── Main component ──────────────────────────────────
export default function ReconstructionView({ reconstruction, partial }) {
// Handle both v0.2 direct object and wrapped result formats
const data = reconstruction;
if (partial) {
return (
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-3 text-sm text-yellow-800">
Partial result some fields failed validation. Showing what was accepted.
Partial result some fields failed validation. Showing what was
accepted.
</div>
);
}
const categories = Object.entries(categoryLabels).map(([key, label]) => ({
key,
label,
items: reconstruction[key],
}));
return (
<div className="space-y-1">
<h2 className="mb-3 text-lg font-semibold">Reconstruction</h2>
{categories.map(({ key, label, items }) => (
<div key={key} className="mb-4 rounded border border-gray-200 bg-white p-4">
<h3 className="mb-2 text-sm font-medium text-gray-600">{label}</h3>
<ItemList items={items} />
</div>
))}
<div className="space-y-4">
{/* Classification first */}
{data.inputClassification && (
<ClassificationDisplay classification={data.inputClassification} />
)}
{/* Summary */}
{data.reconstruction?.summary && (
<SummaryDisplay reconstruction={data.reconstruction} />
)}
{/* Key differences */}
{data.reconstruction?.differences && (
<ItemList
title="Key Differences"
items={data.reconstruction.differences}
/>
)}
{/* Unexplained transitions */}
{data.reconstruction?.unexplainedTransitions &&
data.reconstruction.unexplainedTransitions.length > 0 && (
<ItemList
title="Unexplained Transitions"
items={data.reconstruction.unexplainedTransitions}
renderExtra={(i) =>
i.entity && (
<p className="mt-1 text-xs text-gray-500">Entity: {i.entity}</p>
)
}
/>
)}
{/* Contradictions */}
{data.reconstruction?.contradictions &&
data.reconstruction.contradictions.length > 0 && (
<ItemList
title="Contradictions"
items={data.reconstruction.contradictions}
/>
)}
{/* Important unknowns */}
{data.reconstruction?.importantUnknowns &&
data.reconstruction.importantUnknowns.length > 0 && (
<ItemList
title="Important Unknowns"
items={data.reconstruction.importantUnknowns}
/>
)}
{/* Plausible interpretations */}
{data.reconstruction?.plausibleInterpretations &&
data.reconstruction.plausibleInterpretations.length > 0 && (
<InterpretationsDisplay
interpretations={data.reconstruction.plausibleInterpretations}
/>
)}
{/* Secondary reconstruction categories (actors, systems, etc.) */}
{data.reconstruction?.actors && data.reconstruction.actors.length > 0 && (
<ItemList title="Actors" items={data.reconstruction.actors} />
)}
{data.reconstruction?.systemsOrObjects &&
data.reconstruction.systemsOrObjects.length > 0 && (
<ItemList
title="Systems / Objects"
items={data.reconstruction.systemsOrObjects}
/>
)}
{data.reconstruction?.expectedStates &&
data.reconstruction.expectedStates.length > 0 && (
<ItemList
title="Expected States"
items={data.reconstruction.expectedStates}
/>
)}
{data.reconstruction?.observedStates &&
data.reconstruction.observedStates.length > 0 && (
<ItemList
title="Observed States"
items={data.reconstruction.observedStates}
/>
)}
{data.reconstruction?.knownTransitions &&
data.reconstruction.knownTransitions.length > 0 && (
<ItemList
title="Known Transitions"
items={data.reconstruction.knownTransitions}
renderExtra={(i) => (
<div className="mt-1 text-xs text-gray-500">
{i.entity && <span>Entity: {i.entity} · </span>}
From &ldquo;{i.previousState}&rdquo; To &ldquo;{i.currentState}&rdquo; (&quot;{i.explanationStatus}&quot;)
</div>
)}
/>
)}
{/* Next question — prominent */}
<NextQuestionDisplay question={data.nextQuestion} />
{/* Evidence */}
{data.evidence && <EvidenceDisplay evidence={data.evidence} />}
</div>
);
}
+50 -22
View File
@@ -8,7 +8,7 @@ const MAX_LENGTH = 10000;
export default function ScenarioForm() {
const [scenario, setScenario] = useState("");
const [status, setStatus] = useState("idle"); // idle | loading | error | success
const [status, setStatus] = useState("idle"); // idle | loading | error | success | partial
const [result, setResult] = useState(null);
const textareaRef = useRef(null);
@@ -29,6 +29,10 @@ export default function ScenarioForm() {
if (res.ok && data.validationStatus === "valid") {
setStatus("success");
setResult(data);
} else if (data.success) {
// Success in analysis but validation may be partial
setStatus("success");
setResult(data);
} else {
setStatus("error");
setResult(data);
@@ -39,8 +43,13 @@ export default function ScenarioForm() {
}
};
// Always show diagnostics when there's a result (even if validation failed)
const hasDiagnostics = result && (result.reconstruction || result.modelName || result.responseDurationMs !== undefined);
// Determine if we have meaningful content to display
const hasClassification = result?.inputClassification;
const hasReconstruction = result?.reconstruction;
const hasNextQuestion = result?.nextQuestion;
const hasEvidence = result?.evidence && result.evidence.length > 0;
const hasMeaningfulContent =
hasClassification || hasReconstruction || hasNextQuestion || hasEvidence;
return (
<div className="space-y-6">
@@ -54,7 +63,9 @@ export default function ScenarioForm() {
className="w-full rounded-lg border border-gray-300 px-4 py-3 text-sm focus:border-gray-500 focus:outline-none focus:ring-2 focus:ring-gray-400"
/>
<div className="flex items-center justify-between">
<span className="text-xs text-gray-400">{scenario.length}/{MAX_LENGTH}</span>
<span className="text-xs text-gray-400">
{scenario.length}/{MAX_LENGTH}
</span>
<button
type="submit"
disabled={status === "loading" || !scenario.trim()}
@@ -65,6 +76,7 @@ export default function ScenarioForm() {
</div>
</form>
{/* Error state */}
{status === "error" && (
<div className="space-y-3">
{result?.error && (
@@ -72,35 +84,51 @@ export default function ScenarioForm() {
Error: {result.error}
</div>
)}
{hasDiagnostics && result?.modelName && (
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1.5 text-sm">
<dt className="text-gray-500">Model</dt>
<dd>{result.modelName}</dd>
<dt className="text-gray-500">Duration</dt>
<dd>{result.responseDurationMs != null ? `${result.responseDurationMs}ms` : "?"}</dd>
</dl>
{/* Show partial content even on validation failure */}
{(hasClassification || hasReconstruction) && (
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
Partial result some fields failed validation. Showing what was
accepted.
</div>
)}
{hasReconstruction && (
<ReconstructionView reconstruction={result} partial />
)}
</div>
)}
{status === "success" && result?.reconstruction && (
{/* Success state */}
{status === "success" && hasMeaningfulContent && (
<div className="space-y-4">
<ReconstructionView reconstruction={result.reconstruction} />
<DiagnosticsView result={result} />
<ReconstructionView reconstruction={result} />
</div>
)}
{status === "error" && result?.reconstruction && (
<div className="space-y-3">
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
Partial result some fields failed validation. Showing what was accepted.
</div>
<ReconstructionView reconstruction={result.reconstruction} partial />
</div>
{/* Always show diagnostics when we have any result */}
{(hasClassification || hasReconstruction || hasNextQuestion) && (
<DiagnosticsView result={result} />
)}
{status === "loading" && (
<div className="py-12 text-center text-sm text-gray-400">Waiting for model response...</div>
<div className="py-12 text-center text-sm text-gray-400">
Waiting for model response...
</div>
)}
{/* Empty state */}
{status === "idle" && (
<div className="rounded-lg border border-dashed border-gray-300 bg-gray-50 px-6 py-8 text-center">
<p className="text-sm text-gray-400">
Enter a scenario above and click Analyse to begin.
</p>
</div>
)}
{/* Invalid result with no partial data */}
{status === "error" && !result?.error && !hasMeaningfulContent && (
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
Validation failed no structured output was produced.
</div>
)}
</div>
);
+182
View File
@@ -0,0 +1,182 @@
/**
* Core analysis pipeline — shared by API routes and evaluation harness.
* Calls the provider, parses output, validates against Zod schemas (v0.2 first, v0.1 fallback).
*/
import { getConfig } from "../lib/config.js";
import { getProvider } from "../lib/llm/provider.js";
import { buildPrompt, PROMPT_VERSIONS } from "../lib/reconstruction/prompt.js";
import {
reconstructionV2Schema,
reconstructionSchema as reconstructionV1Schema,
} from "../lib/reconstruction/schema.js";
const MAX_SCENARIO_LENGTH = 10000;
const DEFAULT_PROMPT_VERSION = "v0.2";
/**
* Analyse a scenario string through the full pipeline.
* @param {string} scenario - The scenario text to analyse
* @param {object} [opts]
* @param {"v0.1" | "v0.2"} [opts.promptVersion="v0.2"] - Prompt version to use
* @returns {Promise<object>} Analysis result with diagnostics
*/
export async function analyseScenario(scenario, opts = {}) {
const startTime = Date.now();
// ── Input validation ───────────────────────────────
if (typeof scenario !== "string") {
return buildErrorResponse("Input must be a string", startTime);
}
const trimmed = scenario.trim();
if (trimmed.length === 0) {
return buildErrorResponse("Scenario cannot be empty", startTime);
}
if (trimmed.length > MAX_SCENARIO_LENGTH) {
return buildErrorResponse(`Scenario must be under ${MAX_SCENARIO_LENGTH} characters`, startTime);
}
// ── Configuration check ────────────────────────────
const configResult = getConfig();
if (!configResult.ok) {
return buildErrorResponse("Invalid server configuration", startTime, "500");
}
const { OLLAMA_BASE_URL: _ignored, OLLAMA_MODEL } = configResult.config;
const promptVersion = opts.promptVersion || DEFAULT_PROMPT_VERSION;
// ── Build prompt ───────────────────────────────────
let promptObj;
try {
promptObj = await buildPrompt(trimmed, promptVersion);
} catch (e) {
return buildErrorResponse(`Failed to build prompt: ${e.message}`, startTime);
}
// ── Call provider ──────────────────────────────────
const provider = getProvider();
let rawResponse;
try {
rawResponse = await provider.generateReconstruction(promptObj.prompt, OLLAMA_MODEL);
} catch (e) {
return buildErrorResponse(
e.message || "Provider error during analysis",
Date.now() - startTime
);
}
const duration = Date.now() - startTime;
// Try to capture raw response for diagnostics
let rawResponseStr;
try {
rawResponseStr = JSON.stringify(rawResponse);
} catch {
rawResponseStr = String(rawResponse).slice(0, 2000);
}
// ── Validate against v0.2 schema (preferred) ──────
const resultV2 = tryValidateAgainstSchema(rawResponse, reconstructionV2Schema);
if (resultV2.valid) {
return buildSuccessResultV2(resultV2.data, OLLAMA_MODEL, duration, promptVersion);
}
// ── Fallback to v0.1 schema ────────────────────────
const resultV1 = tryValidateAgainstSchema(rawResponse, reconstructionV1Schema);
if (resultV1.valid) {
return buildSuccessResultV1(resultV1.data, OLLAMA_MODEL, duration, promptVersion);
}
// ── Neither schema matched — partial failure ───────
return buildPartialResult(
rawResponseStr?.slice(0, 2000),
resultV2.error ?? resultV1.error,
OLLAMA_MODEL,
duration,
promptVersion
);
}
/** Attempt validation against a Zod schema */
function tryValidateAgainstSchema(data, schema) {
if (!schema.safeParse) {
return { valid: false, error: new Error("Schema does not support safeParse") };
}
const result = schema.safeParse(data);
return result.success ? { valid: true, data: result.data } : { valid: false, error: result.error };
}
// ── Result builders ──────────────────────────────────
function buildErrorResponse(message, elapsed, statusCode = 500) {
return {
success: false,
error: message,
modelName: null,
responseDurationMs: elapsed,
validationStatus: "invalid",
rawResponse: null,
promptVersion: null,
statusCode,
};
}
function buildSuccessResultV2(data, model, duration, version) {
return {
success: true,
validationStatus: "valid",
modelName: model,
responseDurationMs: duration,
rawResponse: JSON.stringify(data).slice(0, 3000),
promptVersion: version,
inputClassification: data.inputClassification,
reconstruction: data.reconstruction,
evidence: data.evidence,
nextQuestion: data.nextQuestion,
errors: undefined,
};
}
function buildSuccessResultV1(data, model, duration, version) {
return {
success: true,
validationStatus: "valid",
modelName: model,
responseDurationMs: duration,
rawResponse: JSON.stringify(data).slice(0, 3000),
promptVersion: version,
inputClassification: null,
reconstruction: data,
evidence: undefined,
nextQuestion: undefined,
errors: undefined,
};
}
function buildPartialResult(rawResp, error, model, duration, version) {
let errors = [];
if (error && typeof error.flatten === "function") {
errors = error.flatten().fieldErrors
? Object.entries(error.flatten().fieldErrors).flatMap(([k, v]) => [`${k}: ${v.join(", ")}`])
: [String(error)];
} else if (error) {
errors = [String(error).slice(0, 500)];
}
return {
success: false,
validationStatus: "invalid",
modelName: model,
responseDurationMs: duration,
rawResponse: rawResp?.slice(0, 2000),
promptVersion: version,
inputClassification: null,
reconstruction: null,
evidence: undefined,
nextQuestion: undefined,
errors,
};
}
export { PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION };
+3 -5
View File
@@ -93,11 +93,9 @@ async function detectChatSupport(baseUrl) {
class OllamaLlmProvider {
async generateReconstruction(scenario, modelName) {
const { buildPrompt } = await import("@/lib/reconstruction/prompt");
let rawPrompt = buildPrompt(scenario);
// Stronger JSON hint since we can't use format:json on older Ollama
const prompt = rawPrompt + `\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.`;
// scenario is ALREADY a fully-built prompt text (built by analyseScenario).
// Do NOT call buildPrompt() again — that would double-wrap the prompt.
const prompt = scenario;
const baseUrl = process.env.OLLAMA_BASE_URL;
if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");
+48 -1
View File
@@ -1,4 +1,16 @@
export function buildPrompt(scenario) {
import { promises as fs } from "node:fs";
import { fileURLToPath } from "node:url";
import { dirname, join } from "node:path";
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const PROMPTS_DIR = join(__dirname, "../../prompts");
/** Available prompt versions */
export const PROMPT_VERSIONS = ["v0.1", "v0.2"];
/** Build a v0.1 (extraction-only) prompt inline for backward compatibility */
function buildV1Prompt(scenario) {
return `You are a neutral analyst performing an evidence-based reconstruction of the following scenario.
Rules:
@@ -29,3 +41,38 @@ Return valid JSON matching this structure exactly:
Return ONLY the JSON object. No markdown, no explanation, no preamble.`;
}
/** Load a versioned prompt from disk and substitute {{SCENARIO}} */
async function buildV2Prompt(scenario) {
try {
const content = await fs.readFile(
join(PROMPTS_DIR, "reconstruct-v0.2.md"),
"utf-8",
);
return content.replace("{{SCENARIO}}", scenario);
} catch {
// Fall back to v0.1 prompt if v0.2 file is missing
return buildV1Prompt(scenario);
}
}
/**
* Build an analysis prompt for the given version.
* @param {"v0.1" | "v0.2"} [version="v0.2"]
* @returns {Promise<{prompt: string, version: string}>}
*/
export async function buildPrompt(scenario, version = "v0.2") {
let prompt;
switch (version) {
case "v0.1":
prompt = buildV1Prompt(scenario);
break;
default: // v0.2
prompt = await buildV2Prompt(scenario);
break;
}
const strongJsonHint =
"\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.";
return { prompt: prompt + strongJsonHint, version };
}
+183 -16
View File
@@ -1,38 +1,59 @@
import { z } from "zod";
const confidenceEnum = z.enum(["low", "medium", "high"]);
// ──────────────────────────────────────────────
// Shared enums (v0.1 & v0.2)
// ──────────────────────────────────────────────
const itemSchema = z.object({
export const confidenceEnum = z.enum(["low", "medium", "high"]);
const importanceEnum = z.enum([
"incidental",
"supporting",
"important",
"critical",
]);
const expectedInfoValueEnum = z.enum(["low", "medium", "high"]);
// ──────────────────────────────────────────────
// v0.1 — extraction-only schema (preserved)
// ──────────────────────────────────────────────
const confidenceEnumV1 = z.enum(["low", "medium", "high"]);
const itemSchemaV1 = z.object({
id: z.string().min(1),
description: z.string().min(1),
confidence: confidenceEnum,
confidence: confidenceEnumV1,
});
export const reconstructionSchema = z.object({
observations: z.array(itemSchema),
observations: z.array(itemSchemaV1),
reportedClaims: z.array(
itemSchema.extend({
attributedTo: z.union([z.string().min(1), z.null()]).optional().nullable(),
})
itemSchemaV1.extend({
attributedTo: z
.union([z.string().min(1), z.null()])
.optional()
.nullable(),
}),
),
assumptions: z.array(itemSchema),
entities: z.array(itemSchema),
assumptions: z.array(itemSchemaV1),
entities: z.array(itemSchemaV1),
transitions: z.array(
itemSchema.extend({
itemSchemaV1.extend({
entity: z.string().min(1),
previousState: z.string().min(1),
currentState: z.string().min(1),
explanationStatus: z.string().min(1),
})
}),
),
expectedButMissing: z.array(itemSchema),
presentButUnexpected: z.array(itemSchema),
contradictions: z.array(itemSchema),
openUncertainties: z.array(itemSchema),
expectedButMissing: z.array(itemSchemaV1),
presentButUnexpected: z.array(itemSchemaV1),
contradictions: z.array(itemSchemaV1),
openUncertainties: z.array(itemSchemaV1),
});
// v0.1 analyse response (used internally)
export const analyseResponseSchema = z.object({
reconstruction: reconstructionSchema,
reconstruction: z.union([reconstructionSchema, z.null()]),
modelName: z.string(),
responseDurationMs: z.number(),
validationStatus: z.enum(["valid", "partial", "invalid"]),
@@ -48,6 +69,141 @@ export const healthResponseSchema = z.object({
error: z.string().nullable(),
});
// ──────────────────────────────────────────────
// v0.2 — reasoning classification + reconstruction
// ──────────────────────────────────────────────
export const inputTypes =
/** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").INPUT_TYPE_VALUE>} */ (
z.enum([
"observed_problem",
"unexplained_change",
"contradiction",
"decision_request",
"causal_claim",
"reported_claim",
"fault_report",
"ambiguous_statement",
"question",
"desired_outcome",
"insufficient_context",
"other",
])
);
export const reasoningModes =
/** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").REASONING_MODE_VALUE>} */ (
z.enum([
"establish_baseline",
"identify_difference",
"reconstruct_transition",
"decompose_aggregate",
"validate_measurement",
"validate_claim",
"investigate_contradiction",
"clarify_meaning",
"decision_support",
"fault_investigation",
"identify_missing_information",
"test_possible_explanations",
"other",
])
);
const evidenceRecordSchema = z.object({
id: z.string().min(1),
description: z.string().min(1),
evidenceType: z.enum([
"direct_observation",
"reported_statement",
"interpretation",
"assumption",
"inferred_relationship",
]),
source: z.string().optional(),
attribution: z.string().nullable().optional(),
confidence: confidenceEnum,
importance: importanceEnum,
});
const reconstructionSchemaV2 = z.object({
summary: z.string().min(1),
actors: z.array(itemSchemaV1),
systemsOrObjects: z.array(itemSchemaV1),
expectedStates: z.array(itemSchemaV1),
observedStates: z.array(itemSchemaV1),
differences: z.array(itemSchemaV1),
knownTransitions: z.array(
itemSchemaV1.extend({
entity: z.string().min(1),
previousState: z.string().min(1),
currentState: z.string().min(1),
explanationStatus: z.string().min(1),
}),
),
unexplainedTransitions: z.array(
itemSchemaV1.extend({
entity: z.string().min(1).optional(),
previousState: z.string().min(1).optional(),
currentState: z.string().min(1).optional(),
}),
),
contradictions: z.array(itemSchemaV1),
importantUnknowns: z.array(itemSchemaV1),
plausibleInterpretations: z.array(
z.object({
id: z.string().min(1),
description: z.string().min(1),
supportingEvidenceIds: z.array(z.string()),
assumptionsRequired: z.array(z.string()).optional().default([]),
confidence: confidenceEnum,
}),
),
});
const inputClassificationSchema = z.object({
primaryType: inputTypes,
secondaryTypes: z.array(inputTypes).optional().default([]),
reasoningModes: z.array(reasoningModes).optional().default([]),
classificationReason: z.string().min(1),
confidence: confidenceEnum,
});
const nextQuestionSchema = z.object({
id: z.string().min(1),
question: z.string().min(1),
targets: z.array(z.string()),
reason: z.string().min(1),
expectedInformationValue: expectedInfoValueEnum,
reasoningMode: reasoningModes.optional().default("other"),
});
// v0.2 complete analysis response (what the model produces)
export const reconstructionV2Schema = z.object({
inputClassification: inputClassificationSchema,
reconstruction: reconstructionSchemaV2,
evidence: z.array(evidenceRecordSchema),
nextQuestion: nextQuestionSchema,
});
// Outer wrapper for API return (includes diagnostics + v0.2 data)
export const analyseResponseV2Schema = z.object({
inputClassification: inputClassificationSchema.optional(),
reconstruction: reconstructionSchemaV2.optional().nullable(),
evidence: z.array(evidenceRecordSchema).optional(),
nextQuestion: nextQuestionSchema.optional(),
modelName: z.string(),
responseDurationMs: z.number(),
validationStatus: z.enum(["valid", "partial", "invalid"]),
rawResponse: z.string().optional(),
errors: z.array(z.string()).optional(),
promptVersion: z.string().optional(),
});
// ──────────────────────────────────────────────
// Parsing helpers
// ──────────────────────────────────────────────
export function parseReconstruction(raw) {
if (typeof raw === "string") {
try {
@@ -58,3 +214,14 @@ export function parseReconstruction(raw) {
}
return reconstructionSchema.parse(raw);
}
export function parseReconstructionV2(raw) {
if (typeof raw === "string") {
try {
raw = JSON.parse(raw);
} catch {
throw new SyntaxError("Model response is not valid JSON");
}
}
return reconstructionV2Schema.parse(raw);
}
+1 -1
View File
@@ -1,7 +1,7 @@
{
"type": "module",
"name": "confidence-engine",
"version": "0.1.0",
"version": "0.2.0-experimental",
"private": true,
"description": "Experimental prototype for evidence-based situation reconstruction using local LLMs",
"scripts": {
+122
View File
@@ -0,0 +1,122 @@
You are a neutral analyst performing evidence-based situation reconstruction.
## Rules
1. Do NOT invent facts, context or causes. Only include information present in the scenario or clearly implied.
2. First determine what kind of input has been supplied. Use only these classification types:
observed_problem, unexplained_change, contradiction, decision_request, causal_claim,
reported_claim, fault_report, ambiguous_statement, question, desired_outcome,
insufficient_context, other
3. Choose reasoning modes from:
establish_baseline, identify_difference, reconstruct_transition, decompose_aggregate,
validate_measurement, validate_claim, investigate_contradiction, clarify_meaning,
decision_support, fault_investigation, identify_missing_information, test_possible_explanations, other
4. Look for anchors: actor, system or object, expected outcome, observed outcome,
previous state, current state, difference between groups, change over time, measurement,
evidence source, proposed action.
5. Identify meaningful differences (e.g., some succeed while others fail; revenue rises while cash falls).
6. Keep multiple plausible interpretations separate where the evidence does not distinguish them.
7. Distinguish: what was said / what it may mean / why it may have been said.
8. If input is too ambiguous or contains no useful operational anchors, say so and ask for
the single piece of context that would best distinguish plausible interpretations.
## Confidence scale
- low — weak evidence, speculation, or missing information
- medium — reasonable inference from available evidence
- high — strong evidence, direct observation, or confirmed fact
## Importance scale (evidence records)
- incidental — minor detail, unlikely to affect conclusions
- supporting — adds context but not critical
- important — materially affects understanding of the situation
- critical — essential to resolving the situation; without it conclusions cannot be drawn
## Expected information value (next question)
- low — marginally useful even if answered
- medium — meaningfully clarifies the situation
- high — would significantly distinguish between plausible explanations or fill a gap in understanding
## Next question selection criteria
Prefer questions that:
- clarify a major difference
- establish a baseline
- explain an important transition
- test an unsupported claim
- distinguish between plausible explanations
- request measurable evidence
- identify who or what is affected
- establish timing
Avoid questions that:
- have already been answered
- assume a cause
- jump to a solution
- ask about motive before the observable situation is understood
- focus on incidental wording
- are too broad to produce useful information
- combine many unrelated questions
## Output format — return this exact JSON structure
Return a JSON object with exactly these four top-level keys (use **camelCase**):
```json
{
"inputClassification": {
"primaryType": "<one of: observed_problem, unexplained_change, contradiction, decision_request, causal_claim, reported_claim, fault_report, ambiguous_statement, question, desired_outcome, insufficient_context, other>",
"secondaryTypes": ["<optional additional types from the same list>"],
"reasoningModes": ["<one or more of: establish_baseline, identify_difference, reconstruct_transition, decompose_aggregate, validate_measurement, validate_claim, investigate_contradiction, clarify_meaning, decision_support, fault_investigation, identify_missing_information, test_possible_explanations, other>"],
"classificationReason": "<brief explanation of why you chose the primary type>",
"confidence": "<low | medium | high>"
},
"reconstruction": {
"summary": "<one-sentence overview of the situation>",
"actors": [{"id": "<any unique string>", "description": "...", "confidence": "<low|medium|high>"}],
"systemsOrObjects": [{"id": "<any unique string>", "description": "...", "confidence": "<low|medium|high>"}],
"expectedStates": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
"observedStates": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
"differences": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
"knownTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}],
"unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "..."}],
"contradictions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
"importantUnknowns": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
"plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": ["<ids that support this interpretation>"], "assumptionsRequired": [], "confidence": "<low|medium|high>"}]
},
"evidence": [
{
"id": "<any unique string>",
"description": "...",
"evidenceType": "<direct_observation | reported_statement | interpretation | assumption | inferred_relationship>",
"source": "<optional — who/where this came from>",
"attribution": null,
"confidence": "<low | medium | high>",
"importance": "<incidental | supporting | important | critical>"
}
],
"nextQuestion": {
"id": "<any unique string>",
"question": "<one precise question>",
"targets": ["<what this question targets — e.g. 'actor', 'system', 'expectedOutcome'>"],
"reason": "<why answering this is important>",
"expectedInformationValue": "<low | medium | high>",
"reasoningMode": "<optional reasoning mode from the list above>"
}
}
```
CRITICAL RULES for JSON output:
1. Use **exactly** the key names shown above (camelCase, no snake_case).
2. The four top-level keys must be: `inputClassification`, `reconstruction`, `evidence`, `nextQuestion`.
3. Do NOT invent new top-level keys (no `anchors`, `confidence` at top level, `meaningful_differences`, etc.).
4. Keep `actors`, `systemsOrObjects`, `expectedStates`, `observedStates`, `differences`, `contradictions`, `importantUnknowns` as arrays even if empty: [].
5. Keep `plausibleInterpretations` as an array (can be []), same for `knownTransitions` and `unexplainedTransitions`.
6. Each object in arrays must have at least `id`, `description`, `confidence`.
Scenario:
{{SCENARIO}}
Return ONLY the JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.