diff --git a/.gitignore b/.gitignore
index a898f55..dd5091d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -34,3 +34,8 @@ Thumbs.db
npm-debug.log*
yarn-debug.log*
yarn-error.log*
+
+# Generated evaluation artifacts (regenerated each run)
+evaluation-results/
+provider-debug-results/
+tests-results/
diff --git a/app/api/analyse/route.js b/app/api/analyse/route.js
index 31ac1e1..fb9563c 100644
--- a/app/api/analyse/route.js
+++ b/app/api/analyse/route.js
@@ -1,101 +1,49 @@
-import { getConfig } from "@/lib/config";
-import { getProvider } from "@/lib/llm/provider";
-import { reconstructionSchema } from "@/lib/reconstruction/schema";
-
-const MAX_SCENARIO_LENGTH = 10000;
+import {
+ analyseScenario,
+ PROMPT_VERSIONS,
+ DEFAULT_PROMPT_VERSION,
+} from "@/lib/analysis";
export async function POST(request) {
- const startTime = Date.now();
- let rawResponse = null;
-
try {
const body = await request.json();
-
+
if (!body.scenario || typeof body.scenario !== "string") {
return Response.json(
{ error: "Request must include a 'scenario' string field" },
- { status: 400 }
+ { status: 400 },
);
}
- const trimmed = body.scenario.trim();
-
- if (trimmed.length === 0) {
+ // Optional prompt version override
+ let promptVersion = DEFAULT_PROMPT_VERSION;
+ if (body.promptVersion && PROMPT_VERSIONS.includes(body.promptVersion)) {
+ promptVersion = body.promptVersion;
+ }
+
+ const result = await analyseScenario(body.scenario, { promptVersion });
+
+ if (!result.success) {
return Response.json(
- { error: "Scenario cannot be empty" },
- { status: 400 }
+ { ...result, reconstruction: result.reconstruction || null },
+ { status: Number(result.statusCode) || 500 },
);
}
- if (trimmed.length > MAX_SCENARIO_LENGTH) {
- return Response.json(
- { error: `Scenario must be under ${MAX_SCENARIO_LENGTH} characters` },
- { status: 400 }
- );
- }
-
- const configResult = getConfig();
- if (!configResult.ok) {
- return Response.json(
- { error: "Invalid server configuration" },
- { status: 500 }
- );
- }
-
- const { OLLAMA_BASE_URL, OLLAMA_MODEL } = configResult.config;
- const provider = getProvider();
-
- // Attempt parse to capture raw for debugging
- let reconstruction;
- try {
- reconstruction = await provider.generateReconstruction(trimmed, OLLAMA_MODEL);
- } catch (e) {
- return Response.json(
- {
- error: e.message || "Unknown server error",
- responseDurationMs: Date.now() - startTime,
- modelName: OLLAMA_MODEL,
- validationStatus: "invalid",
- },
- { status: 500 }
- );
- }
-
- // Try to stringify for rawResponse display (safe even if it's already an object)
- try {
- rawResponse = JSON.stringify(reconstruction);
- } catch {
- rawResponse = String(reconstruction).slice(0, 2000);
- }
-
- const duration = Date.now() - startTime;
-
- // Validate with Zod schema
- const validationResult = reconstructionSchema.safeParse(reconstruction);
-
- if (!validationResult.success) {
- return Response.json({
- reconstruction: null,
- modelName: OLLAMA_MODEL,
- responseDurationMs: duration,
- validationStatus: "invalid",
- rawResponse: rawResponse?.slice(0, 2000),
- errors: validationResult.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`),
- });
- }
-
return Response.json({
- reconstruction: validationResult.data,
- modelName: OLLAMA_MODEL,
- responseDurationMs: duration,
- validationStatus: "valid",
- rawResponse: rawResponse?.slice(0, 2000),
+ inputClassification: result.inputClassification,
+ reconstruction: result.reconstruction,
+ evidence: result.evidence,
+ nextQuestion: result.nextQuestion,
+ modelName: result.modelName,
+ responseDurationMs: result.responseDurationMs,
+ validationStatus: result.validationStatus,
+ promptVersion: result.promptVersion,
});
} catch (e) {
- const duration = Date.now() - startTime;
return Response.json(
- { error: e.message || "Unknown server error", responseDurationMs: duration },
- { status: 500 }
+ { error: e.message || "Unknown server error", responseDurationMs: 0 },
+ { status: 500 },
);
}
}
diff --git a/components/diagnostics-view.jsx b/components/diagnostics-view.jsx
index 535c4e4..60a5d05 100644
--- a/components/diagnostics-view.jsx
+++ b/components/diagnostics-view.jsx
@@ -10,17 +10,40 @@ const ValidationIndicator = ({ status }) => {
invalid: "❌ Validation failed",
};
return (
-
+
{labels[status] || status}
);
};
+const validationIcons = {
+ valid: "✅",
+ partial: "⚠️",
+ invalid: "❌",
+};
+
export default function DiagnosticsView({ result }) {
+ if (!result) return null;
+
const metrics = [
{ label: "Model", value: result.modelName || "?" },
- { label: "Duration", value: result.responseDurationMs != null ? `${result.responseDurationMs}ms` : "?" },
- { label: "Validation", value:
},
+ { label: "Provider", value: "Ollama" },
+ { label: "Prompt version", value: result.promptVersion || "?" },
+ {
+ label: "Duration",
+ value:
+ result.responseDurationMs != null
+ ? `${result.responseDurationMs}ms`
+ : "?",
+ },
+ {
+ label: "Validation",
+ value: (
+
+ ),
+ },
];
return (
@@ -35,16 +58,32 @@ export default function DiagnosticsView({ result }) {
))}
+ {/* Collapsed raw output for debugging */}
{result.rawResponse && (
- View raw model response
+ View raw model response (
+ {(result.rawResponse?.length || 0).toLocaleString()} chars)
{result.rawResponse}
)}
+
+ {/* Errors if present */}
+ {result.errors && result.errors.length > 0 && (
+
+
+ Validation errors ({result.errors.length})
+
+
+ {result.errors.map((err, i) => (
+ {err}
+ ))}
+
+
+ )}
);
}
diff --git a/components/reconstruction-view.jsx b/components/reconstruction-view.jsx
index 67e8034..e740631 100644
--- a/components/reconstruction-view.jsx
+++ b/components/reconstruction-view.jsx
@@ -1,15 +1,8 @@
-const categoryLabels = {
- observations: "Direct Observations",
- reportedClaims: "Reported Claims",
- assumptions: "Unsupported Assumptions",
- entities: "Entities",
- transitions: "Transitions",
- expectedButMissing: "Expected But Missing",
- presentButUnexpected: "Present But Unexpected",
- contradictions: "Contradictions",
- openUncertainties: "Open Uncertainties",
-};
+"use client";
+import { useMemo } from "react";
+
+// ── Confidence badge (shared) ────────────────────────
const confidenceColor = {
low: "text-red-600 bg-red-50 border-red-200",
medium: "text-yellow-700 bg-yellow-50 border-yellow-200",
@@ -17,54 +10,401 @@ const confidenceColor = {
};
const ConfidenceBadge = ({ level }) => (
-
+
{level}
);
-function ItemList({ items, renderExtra }) {
- if (!items?.length) return None identified
;
-
+// ── Evidence type labels (shared) ───────────────────
+const evidenceTypeLabels = {
+ direct_observation: "Direct Observation",
+ reported_statement: "Reported Statement",
+ interpretation: "Interpretation",
+ assumption: "Assumption",
+ inferred_relationship: "Inferred Relationship",
+};
+
+const importanceColors = {
+ incidental: "text-gray-500 bg-gray-50 border-gray-200",
+ supporting: "text-blue-700 bg-blue-50 border-blue-200",
+ important: "text-orange-700 bg-orange-50 border-orange-200",
+ critical: "text-red-800 bg-red-50 border-red-300 font-semibold",
+};
+
+const importanceLabels = {
+ incidental: "Incidental",
+ supporting: "Supporting",
+ important: "Important",
+ critical: "Critical",
+};
+
+// ── Input classification display ────────────────────
+function ClassificationDisplay({ classification }) {
+ if (!classification) return null;
+ const p = classification.primaryType || classification.primary_type;
+ const sec =
+ classification.secondaryTypes || classification.secondary_types || [];
+ const modes =
+ classification.reasoningModes || classification.reasoning_modes || [];
+
+ // Normalize camelCase to snake_case for display if needed
+ const primaryLabel = String(p)
+ .replace(/_/g, " ")
+ .replace(/\b\w/g, (c) => c.toUpperCase());
+ const secLabels = sec.map((s) =>
+ s.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()),
+ );
+ const modeLabels = modes.map((m) =>
+ m.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()),
+ );
+
return (
-
- {items.map((item) => (
-
-
- #{item.id}
-
-
- {item.description}
- {renderExtra && renderExtra(item)}
-
- ))}
-
+
+
+ Input Classification
+
+
+ Primary type
+ {primaryLabel}
+ {secLabels.length > 0 && (
+ <>
+ Secondary types
+ {secLabels.join(" · ")}
+ >
+ )}
+ {modeLabels.length > 0 && (
+ <>
+ Reasoning modes
+ {modeLabels.join(" · ")}
+ >
+ )}
+ Classification reason
+
+ {classification.classificationReason ||
+ classification.classification_reason}
+
+ Confidence
+
+
+
+
+
);
}
+// ── Reconstruction summary ──────────────────────────
+function SummaryDisplay({ reconstruction }) {
+ if (!reconstruction?.summary) return null;
+ const summary = reconstruction.summary || reconstruction.Summary;
+ return (
+
+
+ Reconstruction Summary
+
+
{summary}
+
+ );
+}
+
+// ── Generic item list (used for multiple sections) ──
+function ItemList({ title, items, renderExtra }) {
+ const count = items?.length;
+ if (!count) return null; // hide empty sections entirely
+
+ const itemsArr = Array.isArray(items) ? items : [items];
+
+ return (
+
+
+ {title} ({count})
+
+
+
+ );
+}
+
+// ── Plausible interpretations ───────────────────────
+function InterpretationsDisplay({ interpretations }) {
+ if (!interpretations?.length) return null;
+ const arr = Array.isArray(interpretations)
+ ? interpretations
+ : [interpretations];
+
+ return (
+
+
+ Plausible Interpretations ({arr.length})
+
+
+ {arr.map((interp, idx) => (
+
+
+
+ {interp.description}
+
+ {interp.confidence && (
+
+ )}
+
+ {interp.supportingEvidenceIds?.length > 0 && (
+
+ Supporting evidence: {interp.supportingEvidenceIds.join(", ")}
+
+ )}
+ {interp.assumptionsRequired?.length > 0 && (
+
+ Requires assumptions: {interp.assumptionsRequired.join("; ")}
+
+ )}
+
+ ))}
+
+
+ );
+}
+
+// ── Next question (prominent) ───────────────────────
+function NextQuestionDisplay({ question }) {
+ if (!question?.question) return null;
+ const q = question.question || question.Question;
+ const targets = question.targets || question.Targets || [];
+ const reason = question.reason || question.Reason || "";
+ const value =
+ question.expectedInformationValue ||
+ question.expected_information_value ||
+ "medium";
+
+ const valueLabel =
+ { low: "Low", medium: "Medium", high: "High" }[value] || "Medium";
+ const valueColor =
+ {
+ low: "bg-yellow-100 text-yellow-800",
+ medium: "bg-blue-100 text-blue-800",
+ high: "bg-green-100 text-green-800",
+ }[value] || "";
+
+ return (
+
+
+
Next Question
+
+ {valueLabel} value
+
+
+
{q}
+ {targets.length > 0 && (
+
Targets: {targets.join(", ")}
+ )}
+ {reason && (
+
Because: {reason}
+ )}
+
+ );
+}
+
+// ── Evidence list ───────────────────────────────────
+function EvidenceDisplay({ evidence }) {
+ if (!evidence?.length) return null;
+ const arr = Array.isArray(evidence) ? evidence : [evidence];
+
+ const evidenceLabels = {
+ direct_observation: "👁 Direct Observation",
+ reported_statement: "🗣 Reported Statement",
+ interpretation: "💡 Interpretation",
+ assumption: "❓ Assumption",
+ inferred_relationship: "🔗 Inferred Relationship",
+ };
+
+ return (
+
+
+ Supporting Evidence ({arr.length})
+
+
+ {arr.map((item, idx) => (
+
+
+ {item.id && (
+
+ #{item.id}
+
+ )}
+
+ {importanceLabels[item.importance]}
+
+
+ {evidenceLabels[item.evidenceType] || item.evidenceType}
+
+ {item.confidence && }
+
+ {item.description}
+ {(item.source || item.attribution) && (
+
+ Source: {item.source || item.attribution}
+
+ )}
+
+ ))}
+
+
+ );
+}
+
+// ── Main component ──────────────────────────────────
export default function ReconstructionView({ reconstruction, partial }) {
+ // Handle both v0.2 direct object and wrapped result formats
+ const data = reconstruction;
+
if (partial) {
return (
- ⚠ Partial result — some fields failed validation. Showing what was accepted.
+ ⚠ Partial result — some fields failed validation. Showing what was
+ accepted.
);
}
- const categories = Object.entries(categoryLabels).map(([key, label]) => ({
- key,
- label,
- items: reconstruction[key],
- }));
-
return (
-
-
Reconstruction
- {categories.map(({ key, label, items }) => (
-
-
{label}
-
-
- ))}
+
+ {/* Classification first */}
+ {data.inputClassification && (
+
+ )}
+
+ {/* Summary */}
+ {data.reconstruction?.summary && (
+
+ )}
+
+ {/* Key differences */}
+ {data.reconstruction?.differences && (
+
+ )}
+
+ {/* Unexplained transitions */}
+ {data.reconstruction?.unexplainedTransitions &&
+ data.reconstruction.unexplainedTransitions.length > 0 && (
+
+ i.entity && (
+ Entity: {i.entity}
+ )
+ }
+ />
+ )}
+
+ {/* Contradictions */}
+ {data.reconstruction?.contradictions &&
+ data.reconstruction.contradictions.length > 0 && (
+
+ )}
+
+ {/* Important unknowns */}
+ {data.reconstruction?.importantUnknowns &&
+ data.reconstruction.importantUnknowns.length > 0 && (
+
+ )}
+
+ {/* Plausible interpretations */}
+ {data.reconstruction?.plausibleInterpretations &&
+ data.reconstruction.plausibleInterpretations.length > 0 && (
+
+ )}
+
+ {/* Secondary reconstruction categories (actors, systems, etc.) */}
+ {data.reconstruction?.actors && data.reconstruction.actors.length > 0 && (
+
+ )}
+ {data.reconstruction?.systemsOrObjects &&
+ data.reconstruction.systemsOrObjects.length > 0 && (
+
+ )}
+ {data.reconstruction?.expectedStates &&
+ data.reconstruction.expectedStates.length > 0 && (
+
+ )}
+ {data.reconstruction?.observedStates &&
+ data.reconstruction.observedStates.length > 0 && (
+
+ )}
+ {data.reconstruction?.knownTransitions &&
+ data.reconstruction.knownTransitions.length > 0 && (
+ (
+
+ {i.entity && Entity: {i.entity} · }
+ From “{i.previousState}” → To “{i.currentState}” ("{i.explanationStatus}")
+
+ )}
+ />
+ )}
+
+ {/* Next question — prominent */}
+
+
+ {/* Evidence */}
+ {data.evidence && }
);
}
diff --git a/components/scenario-form.jsx b/components/scenario-form.jsx
index 85bdd4c..fb2e7d8 100644
--- a/components/scenario-form.jsx
+++ b/components/scenario-form.jsx
@@ -8,7 +8,7 @@ const MAX_LENGTH = 10000;
export default function ScenarioForm() {
const [scenario, setScenario] = useState("");
- const [status, setStatus] = useState("idle"); // idle | loading | error | success
+ const [status, setStatus] = useState("idle"); // idle | loading | error | success | partial
const [result, setResult] = useState(null);
const textareaRef = useRef(null);
@@ -29,6 +29,10 @@ export default function ScenarioForm() {
if (res.ok && data.validationStatus === "valid") {
setStatus("success");
setResult(data);
+ } else if (data.success) {
+ // Success in analysis but validation may be partial
+ setStatus("success");
+ setResult(data);
} else {
setStatus("error");
setResult(data);
@@ -39,8 +43,13 @@ export default function ScenarioForm() {
}
};
- // Always show diagnostics when there's a result (even if validation failed)
- const hasDiagnostics = result && (result.reconstruction || result.modelName || result.responseDurationMs !== undefined);
+ // Determine if we have meaningful content to display
+ const hasClassification = result?.inputClassification;
+ const hasReconstruction = result?.reconstruction;
+ const hasNextQuestion = result?.nextQuestion;
+ const hasEvidence = result?.evidence && result.evidence.length > 0;
+ const hasMeaningfulContent =
+ hasClassification || hasReconstruction || hasNextQuestion || hasEvidence;
return (
@@ -54,7 +63,9 @@ export default function ScenarioForm() {
className="w-full rounded-lg border border-gray-300 px-4 py-3 text-sm focus:border-gray-500 focus:outline-none focus:ring-2 focus:ring-gray-400"
/>
-
{scenario.length}/{MAX_LENGTH}
+
+ {scenario.length}/{MAX_LENGTH}
+
+ {/* Error state */}
{status === "error" && (
{result?.error && (
@@ -72,35 +84,51 @@ export default function ScenarioForm() {
Error: {result.error}
)}
- {hasDiagnostics && result?.modelName && (
-
- Model
- {result.modelName}
- Duration
- {result.responseDurationMs != null ? `${result.responseDurationMs}ms` : "?"}
-
+ {/* Show partial content even on validation failure */}
+ {(hasClassification || hasReconstruction) && (
+
+ ⚠ Partial result — some fields failed validation. Showing what was
+ accepted.
+
+ )}
+ {hasReconstruction && (
+
)}
)}
- {status === "success" && result?.reconstruction && (
+ {/* Success state */}
+ {status === "success" && hasMeaningfulContent && (
-
-
+
)}
- {status === "error" && result?.reconstruction && (
-
-
- ⚠ Partial result — some fields failed validation. Showing what was accepted.
-
-
-
+ {/* Always show diagnostics when we have any result */}
+ {(hasClassification || hasReconstruction || hasNextQuestion) && (
+
)}
{status === "loading" && (
-
Waiting for model response...
+
+ Waiting for model response...
+
+ )}
+
+ {/* Empty state */}
+ {status === "idle" && (
+
+
+ Enter a scenario above and click Analyse to begin.
+
+
+ )}
+
+ {/* Invalid result with no partial data */}
+ {status === "error" && !result?.error && !hasMeaningfulContent && (
+
+ Validation failed — no structured output was produced.
+
)}
);
diff --git a/lib/analysis.js b/lib/analysis.js
new file mode 100644
index 0000000..8741cf1
--- /dev/null
+++ b/lib/analysis.js
@@ -0,0 +1,182 @@
+/**
+ * Core analysis pipeline — shared by API routes and evaluation harness.
+ * Calls the provider, parses output, validates against Zod schemas (v0.2 first, v0.1 fallback).
+ */
+
+import { getConfig } from "../lib/config.js";
+import { getProvider } from "../lib/llm/provider.js";
+import { buildPrompt, PROMPT_VERSIONS } from "../lib/reconstruction/prompt.js";
+import {
+ reconstructionV2Schema,
+ reconstructionSchema as reconstructionV1Schema,
+} from "../lib/reconstruction/schema.js";
+
+const MAX_SCENARIO_LENGTH = 10000;
+const DEFAULT_PROMPT_VERSION = "v0.2";
+
+/**
+ * Analyse a scenario string through the full pipeline.
+ * @param {string} scenario - The scenario text to analyse
+ * @param {object} [opts]
+ * @param {"v0.1" | "v0.2"} [opts.promptVersion="v0.2"] - Prompt version to use
+ * @returns {Promise
} Analysis result with diagnostics
+ */
+export async function analyseScenario(scenario, opts = {}) {
+ const startTime = Date.now();
+
+ // ── Input validation ───────────────────────────────
+ if (typeof scenario !== "string") {
+ return buildErrorResponse("Input must be a string", startTime);
+ }
+
+ const trimmed = scenario.trim();
+ if (trimmed.length === 0) {
+ return buildErrorResponse("Scenario cannot be empty", startTime);
+ }
+ if (trimmed.length > MAX_SCENARIO_LENGTH) {
+ return buildErrorResponse(`Scenario must be under ${MAX_SCENARIO_LENGTH} characters`, startTime);
+ }
+
+ // ── Configuration check ────────────────────────────
+ const configResult = getConfig();
+ if (!configResult.ok) {
+ return buildErrorResponse("Invalid server configuration", startTime, "500");
+ }
+
+ const { OLLAMA_BASE_URL: _ignored, OLLAMA_MODEL } = configResult.config;
+ const promptVersion = opts.promptVersion || DEFAULT_PROMPT_VERSION;
+
+ // ── Build prompt ───────────────────────────────────
+ let promptObj;
+ try {
+ promptObj = await buildPrompt(trimmed, promptVersion);
+ } catch (e) {
+ return buildErrorResponse(`Failed to build prompt: ${e.message}`, startTime);
+ }
+
+ // ── Call provider ──────────────────────────────────
+ const provider = getProvider();
+ let rawResponse;
+ try {
+ rawResponse = await provider.generateReconstruction(promptObj.prompt, OLLAMA_MODEL);
+ } catch (e) {
+ return buildErrorResponse(
+ e.message || "Provider error during analysis",
+ Date.now() - startTime
+ );
+ }
+
+ const duration = Date.now() - startTime;
+
+ // Try to capture raw response for diagnostics
+ let rawResponseStr;
+ try {
+ rawResponseStr = JSON.stringify(rawResponse);
+ } catch {
+ rawResponseStr = String(rawResponse).slice(0, 2000);
+ }
+
+ // ── Validate against v0.2 schema (preferred) ──────
+ const resultV2 = tryValidateAgainstSchema(rawResponse, reconstructionV2Schema);
+ if (resultV2.valid) {
+ return buildSuccessResultV2(resultV2.data, OLLAMA_MODEL, duration, promptVersion);
+ }
+
+ // ── Fallback to v0.1 schema ────────────────────────
+ const resultV1 = tryValidateAgainstSchema(rawResponse, reconstructionV1Schema);
+ if (resultV1.valid) {
+ return buildSuccessResultV1(resultV1.data, OLLAMA_MODEL, duration, promptVersion);
+ }
+
+ // ── Neither schema matched — partial failure ───────
+ return buildPartialResult(
+ rawResponseStr?.slice(0, 2000),
+ resultV2.error ?? resultV1.error,
+ OLLAMA_MODEL,
+ duration,
+ promptVersion
+ );
+}
+
+/** Attempt validation against a Zod schema */
+function tryValidateAgainstSchema(data, schema) {
+ if (!schema.safeParse) {
+ return { valid: false, error: new Error("Schema does not support safeParse") };
+ }
+ const result = schema.safeParse(data);
+ return result.success ? { valid: true, data: result.data } : { valid: false, error: result.error };
+}
+
+// ── Result builders ──────────────────────────────────
+
+function buildErrorResponse(message, elapsed, statusCode = 500) {
+ return {
+ success: false,
+ error: message,
+ modelName: null,
+ responseDurationMs: elapsed,
+ validationStatus: "invalid",
+ rawResponse: null,
+ promptVersion: null,
+ statusCode,
+ };
+}
+
+function buildSuccessResultV2(data, model, duration, version) {
+ return {
+ success: true,
+ validationStatus: "valid",
+ modelName: model,
+ responseDurationMs: duration,
+ rawResponse: JSON.stringify(data).slice(0, 3000),
+ promptVersion: version,
+ inputClassification: data.inputClassification,
+ reconstruction: data.reconstruction,
+ evidence: data.evidence,
+ nextQuestion: data.nextQuestion,
+ errors: undefined,
+ };
+}
+
+function buildSuccessResultV1(data, model, duration, version) {
+ return {
+ success: true,
+ validationStatus: "valid",
+ modelName: model,
+ responseDurationMs: duration,
+ rawResponse: JSON.stringify(data).slice(0, 3000),
+ promptVersion: version,
+ inputClassification: null,
+ reconstruction: data,
+ evidence: undefined,
+ nextQuestion: undefined,
+ errors: undefined,
+ };
+}
+
+function buildPartialResult(rawResp, error, model, duration, version) {
+ let errors = [];
+ if (error && typeof error.flatten === "function") {
+ errors = error.flatten().fieldErrors
+ ? Object.entries(error.flatten().fieldErrors).flatMap(([k, v]) => [`${k}: ${v.join(", ")}`])
+ : [String(error)];
+ } else if (error) {
+ errors = [String(error).slice(0, 500)];
+ }
+
+ return {
+ success: false,
+ validationStatus: "invalid",
+ modelName: model,
+ responseDurationMs: duration,
+ rawResponse: rawResp?.slice(0, 2000),
+ promptVersion: version,
+ inputClassification: null,
+ reconstruction: null,
+ evidence: undefined,
+ nextQuestion: undefined,
+ errors,
+ };
+}
+
+export { PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION };
diff --git a/lib/llm/provider.js b/lib/llm/provider.js
index 716a8cc..c41a313 100644
--- a/lib/llm/provider.js
+++ b/lib/llm/provider.js
@@ -93,11 +93,9 @@ async function detectChatSupport(baseUrl) {
class OllamaLlmProvider {
async generateReconstruction(scenario, modelName) {
- const { buildPrompt } = await import("@/lib/reconstruction/prompt");
-
- let rawPrompt = buildPrompt(scenario);
- // Stronger JSON hint since we can't use format:json on older Ollama
- const prompt = rawPrompt + `\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.`;
+ // scenario is ALREADY a fully-built prompt text (built by analyseScenario).
+ // Do NOT call buildPrompt() again — that would double-wrap the prompt.
+ const prompt = scenario;
const baseUrl = process.env.OLLAMA_BASE_URL;
if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");
diff --git a/lib/reconstruction/prompt.js b/lib/reconstruction/prompt.js
index 691700d..66edeab 100644
--- a/lib/reconstruction/prompt.js
+++ b/lib/reconstruction/prompt.js
@@ -1,5 +1,17 @@
-export function buildPrompt(scenario) {
- return `You are a neutral analyst performing an evidence-based reconstruction of the following scenario.
+import { promises as fs } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { dirname, join } from "node:path";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const PROMPTS_DIR = join(__dirname, "../../prompts");
+
+/** Available prompt versions */
+export const PROMPT_VERSIONS = ["v0.1", "v0.2"];
+
+/** Build a v0.1 (extraction-only) prompt inline for backward compatibility */
+function buildV1Prompt(scenario) {
+ return `You are a neutral analyst performing an evidence-based reconstruction of the following scenario.
Rules:
1. Do NOT invent facts. Only include information present in the scenario or clearly implied.
@@ -29,3 +41,38 @@ Return valid JSON matching this structure exactly:
Return ONLY the JSON object. No markdown, no explanation, no preamble.`;
}
+
+/** Load a versioned prompt from disk and substitute {{SCENARIO}} */
+async function buildV2Prompt(scenario) {
+ try {
+ const content = await fs.readFile(
+ join(PROMPTS_DIR, "reconstruct-v0.2.md"),
+ "utf-8",
+ );
+ return content.replace("{{SCENARIO}}", scenario);
+ } catch {
+ // Fall back to v0.1 prompt if v0.2 file is missing
+ return buildV1Prompt(scenario);
+ }
+}
+
+/**
+ * Build an analysis prompt for the given version.
+ * @param {"v0.1" | "v0.2"} [version="v0.2"]
+ * @returns {Promise<{prompt: string, version: string}>}
+ */
+export async function buildPrompt(scenario, version = "v0.2") {
+ let prompt;
+ switch (version) {
+ case "v0.1":
+ prompt = buildV1Prompt(scenario);
+ break;
+ default: // v0.2
+ prompt = await buildV2Prompt(scenario);
+ break;
+ }
+
+ const strongJsonHint =
+ "\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.";
+ return { prompt: prompt + strongJsonHint, version };
+}
diff --git a/lib/reconstruction/schema.js b/lib/reconstruction/schema.js
index f94e831..d6f237f 100644
--- a/lib/reconstruction/schema.js
+++ b/lib/reconstruction/schema.js
@@ -1,38 +1,59 @@
import { z } from "zod";
-const confidenceEnum = z.enum(["low", "medium", "high"]);
+// ──────────────────────────────────────────────
+// Shared enums (v0.1 & v0.2)
+// ──────────────────────────────────────────────
-const itemSchema = z.object({
+export const confidenceEnum = z.enum(["low", "medium", "high"]);
+const importanceEnum = z.enum([
+ "incidental",
+ "supporting",
+ "important",
+ "critical",
+]);
+const expectedInfoValueEnum = z.enum(["low", "medium", "high"]);
+
+// ──────────────────────────────────────────────
+// v0.1 — extraction-only schema (preserved)
+// ──────────────────────────────────────────────
+
+const confidenceEnumV1 = z.enum(["low", "medium", "high"]);
+
+const itemSchemaV1 = z.object({
id: z.string().min(1),
description: z.string().min(1),
- confidence: confidenceEnum,
+ confidence: confidenceEnumV1,
});
export const reconstructionSchema = z.object({
- observations: z.array(itemSchema),
+ observations: z.array(itemSchemaV1),
reportedClaims: z.array(
- itemSchema.extend({
- attributedTo: z.union([z.string().min(1), z.null()]).optional().nullable(),
- })
+ itemSchemaV1.extend({
+ attributedTo: z
+ .union([z.string().min(1), z.null()])
+ .optional()
+ .nullable(),
+ }),
),
- assumptions: z.array(itemSchema),
- entities: z.array(itemSchema),
+ assumptions: z.array(itemSchemaV1),
+ entities: z.array(itemSchemaV1),
transitions: z.array(
- itemSchema.extend({
+ itemSchemaV1.extend({
entity: z.string().min(1),
previousState: z.string().min(1),
currentState: z.string().min(1),
explanationStatus: z.string().min(1),
- })
+ }),
),
- expectedButMissing: z.array(itemSchema),
- presentButUnexpected: z.array(itemSchema),
- contradictions: z.array(itemSchema),
- openUncertainties: z.array(itemSchema),
+ expectedButMissing: z.array(itemSchemaV1),
+ presentButUnexpected: z.array(itemSchemaV1),
+ contradictions: z.array(itemSchemaV1),
+ openUncertainties: z.array(itemSchemaV1),
});
+// v0.1 analyse response (used internally)
export const analyseResponseSchema = z.object({
- reconstruction: reconstructionSchema,
+ reconstruction: z.union([reconstructionSchema, z.null()]),
modelName: z.string(),
responseDurationMs: z.number(),
validationStatus: z.enum(["valid", "partial", "invalid"]),
@@ -48,6 +69,141 @@ export const healthResponseSchema = z.object({
error: z.string().nullable(),
});
+// ──────────────────────────────────────────────
+// v0.2 — reasoning classification + reconstruction
+// ──────────────────────────────────────────────
+
+export const inputTypes =
+ /** @type {z.ZodType} */ (
+ z.enum([
+ "observed_problem",
+ "unexplained_change",
+ "contradiction",
+ "decision_request",
+ "causal_claim",
+ "reported_claim",
+ "fault_report",
+ "ambiguous_statement",
+ "question",
+ "desired_outcome",
+ "insufficient_context",
+ "other",
+ ])
+ );
+
+export const reasoningModes =
+ /** @type {z.ZodType} */ (
+ z.enum([
+ "establish_baseline",
+ "identify_difference",
+ "reconstruct_transition",
+ "decompose_aggregate",
+ "validate_measurement",
+ "validate_claim",
+ "investigate_contradiction",
+ "clarify_meaning",
+ "decision_support",
+ "fault_investigation",
+ "identify_missing_information",
+ "test_possible_explanations",
+ "other",
+ ])
+ );
+
+const evidenceRecordSchema = z.object({
+ id: z.string().min(1),
+ description: z.string().min(1),
+ evidenceType: z.enum([
+ "direct_observation",
+ "reported_statement",
+ "interpretation",
+ "assumption",
+ "inferred_relationship",
+ ]),
+ source: z.string().optional(),
+ attribution: z.string().nullable().optional(),
+ confidence: confidenceEnum,
+ importance: importanceEnum,
+});
+
+const reconstructionSchemaV2 = z.object({
+ summary: z.string().min(1),
+ actors: z.array(itemSchemaV1),
+ systemsOrObjects: z.array(itemSchemaV1),
+ expectedStates: z.array(itemSchemaV1),
+ observedStates: z.array(itemSchemaV1),
+ differences: z.array(itemSchemaV1),
+ knownTransitions: z.array(
+ itemSchemaV1.extend({
+ entity: z.string().min(1),
+ previousState: z.string().min(1),
+ currentState: z.string().min(1),
+ explanationStatus: z.string().min(1),
+ }),
+ ),
+ unexplainedTransitions: z.array(
+ itemSchemaV1.extend({
+ entity: z.string().min(1).optional(),
+ previousState: z.string().min(1).optional(),
+ currentState: z.string().min(1).optional(),
+ }),
+ ),
+ contradictions: z.array(itemSchemaV1),
+ importantUnknowns: z.array(itemSchemaV1),
+ plausibleInterpretations: z.array(
+ z.object({
+ id: z.string().min(1),
+ description: z.string().min(1),
+ supportingEvidenceIds: z.array(z.string()),
+ assumptionsRequired: z.array(z.string()).optional().default([]),
+ confidence: confidenceEnum,
+ }),
+ ),
+});
+
+const inputClassificationSchema = z.object({
+ primaryType: inputTypes,
+ secondaryTypes: z.array(inputTypes).optional().default([]),
+ reasoningModes: z.array(reasoningModes).optional().default([]),
+ classificationReason: z.string().min(1),
+ confidence: confidenceEnum,
+});
+
+const nextQuestionSchema = z.object({
+ id: z.string().min(1),
+ question: z.string().min(1),
+ targets: z.array(z.string()),
+ reason: z.string().min(1),
+ expectedInformationValue: expectedInfoValueEnum,
+ reasoningMode: reasoningModes.optional().default("other"),
+});
+
+// v0.2 complete analysis response (what the model produces)
+export const reconstructionV2Schema = z.object({
+ inputClassification: inputClassificationSchema,
+ reconstruction: reconstructionSchemaV2,
+ evidence: z.array(evidenceRecordSchema),
+ nextQuestion: nextQuestionSchema,
+});
+
+// Outer wrapper for API return (includes diagnostics + v0.2 data)
+export const analyseResponseV2Schema = z.object({
+ inputClassification: inputClassificationSchema.optional(),
+ reconstruction: reconstructionSchemaV2.optional().nullable(),
+ evidence: z.array(evidenceRecordSchema).optional(),
+ nextQuestion: nextQuestionSchema.optional(),
+ modelName: z.string(),
+ responseDurationMs: z.number(),
+ validationStatus: z.enum(["valid", "partial", "invalid"]),
+ rawResponse: z.string().optional(),
+ errors: z.array(z.string()).optional(),
+ promptVersion: z.string().optional(),
+});
+
+// ──────────────────────────────────────────────
+// Parsing helpers
+// ──────────────────────────────────────────────
+
export function parseReconstruction(raw) {
if (typeof raw === "string") {
try {
@@ -58,3 +214,14 @@ export function parseReconstruction(raw) {
}
return reconstructionSchema.parse(raw);
}
+
+export function parseReconstructionV2(raw) {
+ if (typeof raw === "string") {
+ try {
+ raw = JSON.parse(raw);
+ } catch {
+ throw new SyntaxError("Model response is not valid JSON");
+ }
+ }
+ return reconstructionV2Schema.parse(raw);
+}
diff --git a/package.json b/package.json
index e3827f3..4f1c4e6 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
{
"type": "module",
"name": "confidence-engine",
- "version": "0.1.0",
+ "version": "0.2.0-experimental",
"private": true,
"description": "Experimental prototype for evidence-based situation reconstruction using local LLMs",
"scripts": {
diff --git a/prompts/reconstruct-v0.2.md b/prompts/reconstruct-v0.2.md
new file mode 100644
index 0000000..6d0f7dd
--- /dev/null
+++ b/prompts/reconstruct-v0.2.md
@@ -0,0 +1,122 @@
+You are a neutral analyst performing evidence-based situation reconstruction.
+
+## Rules
+
+1. Do NOT invent facts, context or causes. Only include information present in the scenario or clearly implied.
+2. First determine what kind of input has been supplied. Use only these classification types:
+ observed_problem, unexplained_change, contradiction, decision_request, causal_claim,
+ reported_claim, fault_report, ambiguous_statement, question, desired_outcome,
+ insufficient_context, other
+3. Choose reasoning modes from:
+ establish_baseline, identify_difference, reconstruct_transition, decompose_aggregate,
+ validate_measurement, validate_claim, investigate_contradiction, clarify_meaning,
+ decision_support, fault_investigation, identify_missing_information, test_possible_explanations, other
+4. Look for anchors: actor, system or object, expected outcome, observed outcome,
+ previous state, current state, difference between groups, change over time, measurement,
+ evidence source, proposed action.
+5. Identify meaningful differences (e.g., some succeed while others fail; revenue rises while cash falls).
+6. Keep multiple plausible interpretations separate where the evidence does not distinguish them.
+7. Distinguish: what was said / what it may mean / why it may have been said.
+8. If input is too ambiguous or contains no useful operational anchors, say so and ask for
+ the single piece of context that would best distinguish plausible interpretations.
+
+## Confidence scale
+
+- low — weak evidence, speculation, or missing information
+- medium — reasonable inference from available evidence
+- high — strong evidence, direct observation, or confirmed fact
+
+## Importance scale (evidence records)
+
+- incidental — minor detail, unlikely to affect conclusions
+- supporting — adds context but not critical
+- important — materially affects understanding of the situation
+- critical — essential to resolving the situation; without it conclusions cannot be drawn
+
+## Expected information value (next question)
+
+- low — marginally useful even if answered
+- medium — meaningfully clarifies the situation
+- high — would significantly distinguish between plausible explanations or fill a gap in understanding
+
+## Next question selection criteria
+
+Prefer questions that:
+- clarify a major difference
+- establish a baseline
+- explain an important transition
+- test an unsupported claim
+- distinguish between plausible explanations
+- request measurable evidence
+- identify who or what is affected
+- establish timing
+
+Avoid questions that:
+- have already been answered
+- assume a cause
+- jump to a solution
+- ask about motive before the observable situation is understood
+- focus on incidental wording
+- are too broad to produce useful information
+- combine many unrelated questions
+
+## Output format — return this exact JSON structure
+
+Return a JSON object with exactly these four top-level keys (use **camelCase**):
+
+```json
+{
+ "inputClassification": {
+ "primaryType": "",
+ "secondaryTypes": [""],
+ "reasoningModes": [""],
+ "classificationReason": "",
+ "confidence": ""
+ },
+ "reconstruction": {
+ "summary": "",
+ "actors": [{"id": "", "description": "...", "confidence": ""}],
+ "systemsOrObjects": [{"id": "", "description": "...", "confidence": ""}],
+ "expectedStates": [{"id": "...", "description": "...", "confidence": ""}],
+ "observedStates": [{"id": "...", "description": "...", "confidence": ""}],
+ "differences": [{"id": "...", "description": "...", "confidence": ""}],
+ "knownTransitions": [{"id": "...", "description": "...", "confidence": "", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}],
+ "unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "", "entity": "...", "previousState": "...", "currentState": "..."}],
+ "contradictions": [{"id": "...", "description": "...", "confidence": ""}],
+ "importantUnknowns": [{"id": "...", "description": "...", "confidence": ""}],
+ "plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": [""], "assumptionsRequired": [], "confidence": ""}]
+ },
+ "evidence": [
+ {
+ "id": "",
+ "description": "...",
+ "evidenceType": "",
+ "source": "",
+ "attribution": null,
+ "confidence": "",
+ "importance": ""
+ }
+ ],
+ "nextQuestion": {
+ "id": "",
+ "question": "",
+ "targets": [""],
+ "reason": "",
+ "expectedInformationValue": "",
+ "reasoningMode": ""
+ }
+}
+```
+
+CRITICAL RULES for JSON output:
+1. Use **exactly** the key names shown above (camelCase, no snake_case).
+2. The four top-level keys must be: `inputClassification`, `reconstruction`, `evidence`, `nextQuestion`.
+3. Do NOT invent new top-level keys (no `anchors`, `confidence` at top level, `meaningful_differences`, etc.).
+4. Keep `actors`, `systemsOrObjects`, `expectedStates`, `observedStates`, `differences`, `contradictions`, `importantUnknowns` as arrays even if empty: [].
+5. Keep `plausibleInterpretations` as an array (can be []), same for `knownTransitions` and `unexplainedTransitions`.
+6. Each object in arrays must have at least `id`, `description`, `confidence`.
+
+Scenario:
+{{SCENARIO}}
+
+Return ONLY the JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.