chore: establish clean v0.2 baseline
Include only the working reconstruction prototype with Ollama integration: - double-wrapping fix (lib/llm/provider.js) - explicit v0.2 JSON output schema (prompts/reconstruct-v0.2.md) - Zod validation layer (lib/reconstruction/schema.js) - shared core analysis path (lib/analysis.js) - prompt versioning infrastructure (lib/reconstruction/prompt.js) - provider abstraction - functioning Ollama provider path - updated API route with centralized analysis - UI components displaying v0.2 data and validation errors - .gitignore rules for generated evaluation artifacts Exclude: evaluator experiments, diagnostic tests, debug scripts, generated artifacts, comparison findings, test data tied to evaluator.
This commit is contained in:
@@ -34,3 +34,8 @@ Thumbs.db
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
# Generated evaluation artifacts (regenerated each run)
|
||||
evaluation-results/
|
||||
provider-debug-results/
|
||||
tests-results/
|
||||
|
||||
+26
-78
@@ -1,101 +1,49 @@
|
||||
import { getConfig } from "@/lib/config";
|
||||
import { getProvider } from "@/lib/llm/provider";
|
||||
import { reconstructionSchema } from "@/lib/reconstruction/schema";
|
||||
|
||||
const MAX_SCENARIO_LENGTH = 10000;
|
||||
import {
|
||||
analyseScenario,
|
||||
PROMPT_VERSIONS,
|
||||
DEFAULT_PROMPT_VERSION,
|
||||
} from "@/lib/analysis";
|
||||
|
||||
export async function POST(request) {
|
||||
const startTime = Date.now();
|
||||
let rawResponse = null;
|
||||
|
||||
try {
|
||||
const body = await request.json();
|
||||
|
||||
if (!body.scenario || typeof body.scenario !== "string") {
|
||||
return Response.json(
|
||||
{ error: "Request must include a 'scenario' string field" },
|
||||
{ status: 400 }
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const trimmed = body.scenario.trim();
|
||||
// Optional prompt version override
|
||||
let promptVersion = DEFAULT_PROMPT_VERSION;
|
||||
if (body.promptVersion && PROMPT_VERSIONS.includes(body.promptVersion)) {
|
||||
promptVersion = body.promptVersion;
|
||||
}
|
||||
|
||||
if (trimmed.length === 0) {
|
||||
const result = await analyseScenario(body.scenario, { promptVersion });
|
||||
|
||||
if (!result.success) {
|
||||
return Response.json(
|
||||
{ error: "Scenario cannot be empty" },
|
||||
{ status: 400 }
|
||||
{ ...result, reconstruction: result.reconstruction || null },
|
||||
{ status: Number(result.statusCode) || 500 },
|
||||
);
|
||||
}
|
||||
|
||||
if (trimmed.length > MAX_SCENARIO_LENGTH) {
|
||||
return Response.json(
|
||||
{ error: `Scenario must be under ${MAX_SCENARIO_LENGTH} characters` },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const configResult = getConfig();
|
||||
if (!configResult.ok) {
|
||||
return Response.json(
|
||||
{ error: "Invalid server configuration" },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
|
||||
const { OLLAMA_BASE_URL, OLLAMA_MODEL } = configResult.config;
|
||||
const provider = getProvider();
|
||||
|
||||
// Attempt parse to capture raw for debugging
|
||||
let reconstruction;
|
||||
try {
|
||||
reconstruction = await provider.generateReconstruction(trimmed, OLLAMA_MODEL);
|
||||
} catch (e) {
|
||||
return Response.json(
|
||||
{
|
||||
error: e.message || "Unknown server error",
|
||||
responseDurationMs: Date.now() - startTime,
|
||||
modelName: OLLAMA_MODEL,
|
||||
validationStatus: "invalid",
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
|
||||
// Try to stringify for rawResponse display (safe even if it's already an object)
|
||||
try {
|
||||
rawResponse = JSON.stringify(reconstruction);
|
||||
} catch {
|
||||
rawResponse = String(reconstruction).slice(0, 2000);
|
||||
}
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
// Validate with Zod schema
|
||||
const validationResult = reconstructionSchema.safeParse(reconstruction);
|
||||
|
||||
if (!validationResult.success) {
|
||||
return Response.json({
|
||||
reconstruction: null,
|
||||
modelName: OLLAMA_MODEL,
|
||||
responseDurationMs: duration,
|
||||
validationStatus: "invalid",
|
||||
rawResponse: rawResponse?.slice(0, 2000),
|
||||
errors: validationResult.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`),
|
||||
});
|
||||
}
|
||||
|
||||
return Response.json({
|
||||
reconstruction: validationResult.data,
|
||||
modelName: OLLAMA_MODEL,
|
||||
responseDurationMs: duration,
|
||||
validationStatus: "valid",
|
||||
rawResponse: rawResponse?.slice(0, 2000),
|
||||
inputClassification: result.inputClassification,
|
||||
reconstruction: result.reconstruction,
|
||||
evidence: result.evidence,
|
||||
nextQuestion: result.nextQuestion,
|
||||
modelName: result.modelName,
|
||||
responseDurationMs: result.responseDurationMs,
|
||||
validationStatus: result.validationStatus,
|
||||
promptVersion: result.promptVersion,
|
||||
});
|
||||
} catch (e) {
|
||||
const duration = Date.now() - startTime;
|
||||
return Response.json(
|
||||
{ error: e.message || "Unknown server error", responseDurationMs: duration },
|
||||
{ status: 500 }
|
||||
{ error: e.message || "Unknown server error", responseDurationMs: 0 },
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,17 +10,40 @@ const ValidationIndicator = ({ status }) => {
|
||||
invalid: "❌ Validation failed",
|
||||
};
|
||||
return (
|
||||
<div className={`flex items-center gap-2 ${styles[status] || "text-gray-500"}`}>
|
||||
<div
|
||||
className={`flex items-center gap-2 ${styles[status] || "text-gray-500"}`}
|
||||
>
|
||||
<span className="font-medium">{labels[status] || status}</span>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
const validationIcons = {
|
||||
valid: "✅",
|
||||
partial: "⚠️",
|
||||
invalid: "❌",
|
||||
};
|
||||
|
||||
export default function DiagnosticsView({ result }) {
|
||||
if (!result) return null;
|
||||
|
||||
const metrics = [
|
||||
{ label: "Model", value: result.modelName || "?" },
|
||||
{ label: "Duration", value: result.responseDurationMs != null ? `${result.responseDurationMs}ms` : "?" },
|
||||
{ label: "Validation", value: <ValidationIndicator status={result.validationStatus || "invalid"} /> },
|
||||
{ label: "Provider", value: "Ollama" },
|
||||
{ label: "Prompt version", value: result.promptVersion || "?" },
|
||||
{
|
||||
label: "Duration",
|
||||
value:
|
||||
result.responseDurationMs != null
|
||||
? `${result.responseDurationMs}ms`
|
||||
: "?",
|
||||
},
|
||||
{
|
||||
label: "Validation",
|
||||
value: (
|
||||
<ValidationIndicator status={result.validationStatus || "invalid"} />
|
||||
),
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
@@ -35,16 +58,32 @@ export default function DiagnosticsView({ result }) {
|
||||
))}
|
||||
</dl>
|
||||
|
||||
{/* Collapsed raw output for debugging */}
|
||||
{result.rawResponse && (
|
||||
<details className="mt-4">
|
||||
<summary className="cursor-pointer text-xs text-gray-500 underline hover:text-gray-700">
|
||||
View raw model response
|
||||
View raw model response (
|
||||
{(result.rawResponse?.length || 0).toLocaleString()} chars)
|
||||
</summary>
|
||||
<pre className="mt-2 max-h-60 overflow-auto rounded bg-gray-900 px-3 py-2 text-xs leading-relaxed text-green-400">
|
||||
{result.rawResponse}
|
||||
</pre>
|
||||
</details>
|
||||
)}
|
||||
|
||||
{/* Errors if present */}
|
||||
{result.errors && result.errors.length > 0 && (
|
||||
<details className="mt-3">
|
||||
<summary className="cursor-pointer text-xs text-red-500 underline hover:text-red-700">
|
||||
Validation errors ({result.errors.length})
|
||||
</summary>
|
||||
<ul className="mt-1 space-y-0.5 text-xs text-red-600">
|
||||
{result.errors.map((err, i) => (
|
||||
<li key={i}>{err}</li>
|
||||
))}
|
||||
</ul>
|
||||
</details>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,15 +1,8 @@
|
||||
const categoryLabels = {
|
||||
observations: "Direct Observations",
|
||||
reportedClaims: "Reported Claims",
|
||||
assumptions: "Unsupported Assumptions",
|
||||
entities: "Entities",
|
||||
transitions: "Transitions",
|
||||
expectedButMissing: "Expected But Missing",
|
||||
presentButUnexpected: "Present But Unexpected",
|
||||
contradictions: "Contradictions",
|
||||
openUncertainties: "Open Uncertainties",
|
||||
};
|
||||
"use client";
|
||||
|
||||
import { useMemo } from "react";
|
||||
|
||||
// ── Confidence badge (shared) ────────────────────────
|
||||
const confidenceColor = {
|
||||
low: "text-red-600 bg-red-50 border-red-200",
|
||||
medium: "text-yellow-700 bg-yellow-50 border-yellow-200",
|
||||
@@ -17,54 +10,401 @@ const confidenceColor = {
|
||||
};
|
||||
|
||||
const ConfidenceBadge = ({ level }) => (
|
||||
<span className={`inline-block rounded-full border px-2 py-0.5 text-xs font-medium ${confidenceColor[level] || "text-gray-600 bg-gray-100"}`}>
|
||||
<span
|
||||
className={`inline-block rounded-full border px-2 py-0.5 text-xs font-medium ${confidenceColor[level] || "text-gray-600 bg-gray-100"}`}
|
||||
>
|
||||
{level}
|
||||
</span>
|
||||
);
|
||||
|
||||
function ItemList({ items, renderExtra }) {
|
||||
if (!items?.length) return <p className="text-sm italic text-gray-400">None identified</p>;
|
||||
// ── Evidence type labels (shared) ───────────────────
|
||||
const evidenceTypeLabels = {
|
||||
direct_observation: "Direct Observation",
|
||||
reported_statement: "Reported Statement",
|
||||
interpretation: "Interpretation",
|
||||
assumption: "Assumption",
|
||||
inferred_relationship: "Inferred Relationship",
|
||||
};
|
||||
|
||||
const importanceColors = {
|
||||
incidental: "text-gray-500 bg-gray-50 border-gray-200",
|
||||
supporting: "text-blue-700 bg-blue-50 border-blue-200",
|
||||
important: "text-orange-700 bg-orange-50 border-orange-200",
|
||||
critical: "text-red-800 bg-red-50 border-red-300 font-semibold",
|
||||
};
|
||||
|
||||
const importanceLabels = {
|
||||
incidental: "Incidental",
|
||||
supporting: "Supporting",
|
||||
important: "Important",
|
||||
critical: "Critical",
|
||||
};
|
||||
|
||||
// ── Input classification display ────────────────────
|
||||
function ClassificationDisplay({ classification }) {
|
||||
if (!classification) return null;
|
||||
const p = classification.primaryType || classification.primary_type;
|
||||
const sec =
|
||||
classification.secondaryTypes || classification.secondary_types || [];
|
||||
const modes =
|
||||
classification.reasoningModes || classification.reasoning_modes || [];
|
||||
|
||||
// Normalize camelCase to snake_case for display if needed
|
||||
const primaryLabel = String(p)
|
||||
.replace(/_/g, " ")
|
||||
.replace(/\b\w/g, (c) => c.toUpperCase());
|
||||
const secLabels = sec.map((s) =>
|
||||
s.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()),
|
||||
);
|
||||
const modeLabels = modes.map((m) =>
|
||||
m.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()),
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="rounded-lg border border-blue-200 bg-blue-50 p-4">
|
||||
<h3 className="mb-2 text-sm font-semibold text-blue-700">
|
||||
Input Classification
|
||||
</h3>
|
||||
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1.5 text-sm">
|
||||
<dt className="text-blue-500">Primary type</dt>
|
||||
<dd className="font-medium">{primaryLabel}</dd>
|
||||
{secLabels.length > 0 && (
|
||||
<>
|
||||
<dt className="text-blue-500 pt-1">Secondary types</dt>
|
||||
<dd>{secLabels.join(" · ")}</dd>
|
||||
</>
|
||||
)}
|
||||
{modeLabels.length > 0 && (
|
||||
<>
|
||||
<dt className="text-blue-500 pt-1">Reasoning modes</dt>
|
||||
<dd>{modeLabels.join(" · ")}</dd>
|
||||
</>
|
||||
)}
|
||||
<dt className="text-blue-500 pt-1">Classification reason</dt>
|
||||
<dd className="italic">
|
||||
{classification.classificationReason ||
|
||||
classification.classification_reason}
|
||||
</dd>
|
||||
<dt className="text-blue-500 pt-1">Confidence</dt>
|
||||
<dd>
|
||||
<ConfidenceBadge level={classification.confidence} />
|
||||
</dd>
|
||||
</dl>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Reconstruction summary ──────────────────────────
|
||||
function SummaryDisplay({ reconstruction }) {
|
||||
if (!reconstruction?.summary) return null;
|
||||
const summary = reconstruction.summary || reconstruction.Summary;
|
||||
return (
|
||||
<div className="rounded-lg border border-gray-200 bg-white p-4">
|
||||
<h3 className="mb-2 text-sm font-semibold text-gray-600">
|
||||
Reconstruction Summary
|
||||
</h3>
|
||||
<p className="text-sm leading-relaxed">{summary}</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Generic item list (used for multiple sections) ──
|
||||
function ItemList({ title, items, renderExtra }) {
|
||||
const count = items?.length;
|
||||
if (!count) return null; // hide empty sections entirely
|
||||
|
||||
const itemsArr = Array.isArray(items) ? items : [items];
|
||||
|
||||
return (
|
||||
<div className="mb-4 rounded-lg border border-gray-200 bg-white p-4">
|
||||
<h3 className="mb-2 text-sm font-semibold text-gray-600">
|
||||
{title} ({count})
|
||||
</h3>
|
||||
<ul className="space-y-2">
|
||||
{items.map((item) => (
|
||||
<li key={item.id} className="rounded border border-gray-200 bg-white px-3 py-2 text-sm">
|
||||
{itemsArr.map((item, idx) => (
|
||||
<li
|
||||
key={item.id || `${title}-${idx}`}
|
||||
className="rounded border border-gray-200 bg-white px-3 py-2 text-sm"
|
||||
>
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="font-mono text-xs text-gray-400">#{item.id}</span>
|
||||
<ConfidenceBadge level={item.confidence} />
|
||||
{item.id && (
|
||||
<span className="font-mono text-xs text-gray-400">
|
||||
#{item.id}
|
||||
</span>
|
||||
)}
|
||||
{item.confidence && <ConfidenceBadge level={item.confidence} />}
|
||||
{item.importance && (
|
||||
<span
|
||||
className={`inline-block rounded-full border px-2 py-0.5 text-xs font-medium ${importanceColors[item.importance] || "text-gray-600 bg-gray-100"}`}
|
||||
>
|
||||
{importanceLabels[item.importance]}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<p className="mt-1">{item.description}</p>
|
||||
{renderExtra && renderExtra(item)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Plausible interpretations ───────────────────────
|
||||
function InterpretationsDisplay({ interpretations }) {
|
||||
if (!interpretations?.length) return null;
|
||||
const arr = Array.isArray(interpretations)
|
||||
? interpretations
|
||||
: [interpretations];
|
||||
|
||||
return (
|
||||
<div className="mb-4 rounded-lg border border-indigo-200 bg-indigo-50 p-4">
|
||||
<h3 className="mb-2 text-sm font-semibold text-indigo-700">
|
||||
Plausible Interpretations ({arr.length})
|
||||
</h3>
|
||||
<ul className="space-y-3">
|
||||
{arr.map((interp, idx) => (
|
||||
<li
|
||||
key={interp.id || `${idx}`}
|
||||
className="rounded border border-indigo-200 bg-white px-3 py-2.5 text-sm leading-relaxed"
|
||||
>
|
||||
<div className="flex items-center gap-2 mb-1">
|
||||
<span className="font-medium text-indigo-600">
|
||||
{interp.description}
|
||||
</span>
|
||||
{interp.confidence && (
|
||||
<ConfidenceBadge level={interp.confidence} />
|
||||
)}
|
||||
</div>
|
||||
{interp.supportingEvidenceIds?.length > 0 && (
|
||||
<p className="text-xs text-gray-500">
|
||||
Supporting evidence: {interp.supportingEvidenceIds.join(", ")}
|
||||
</p>
|
||||
)}
|
||||
{interp.assumptionsRequired?.length > 0 && (
|
||||
<p className="text-xs italic text-gray-500">
|
||||
Requires assumptions: {interp.assumptionsRequired.join("; ")}
|
||||
</p>
|
||||
)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Next question (prominent) ───────────────────────
|
||||
function NextQuestionDisplay({ question }) {
|
||||
if (!question?.question) return null;
|
||||
const q = question.question || question.Question;
|
||||
const targets = question.targets || question.Targets || [];
|
||||
const reason = question.reason || question.Reason || "";
|
||||
const value =
|
||||
question.expectedInformationValue ||
|
||||
question.expected_information_value ||
|
||||
"medium";
|
||||
|
||||
const valueLabel =
|
||||
{ low: "Low", medium: "Medium", high: "High" }[value] || "Medium";
|
||||
const valueColor =
|
||||
{
|
||||
low: "bg-yellow-100 text-yellow-800",
|
||||
medium: "bg-blue-100 text-blue-800",
|
||||
high: "bg-green-100 text-green-800",
|
||||
}[value] || "";
|
||||
|
||||
return (
|
||||
<div className="rounded-lg border-2 border-green-300 bg-green-50 p-5">
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
<h3 className="text-sm font-bold text-green-800">Next Question</h3>
|
||||
<span
|
||||
className={`rounded-full px-2 py-0.5 text-xs font-medium ${valueColor}`}
|
||||
>
|
||||
{valueLabel} value
|
||||
</span>
|
||||
</div>
|
||||
<p className="mb-2 text-base font-medium text-gray-900">{q}</p>
|
||||
{targets.length > 0 && (
|
||||
<p className="text-sm text-gray-600">Targets: {targets.join(", ")}</p>
|
||||
)}
|
||||
{reason && (
|
||||
<p className="text-sm italic text-gray-500">Because: {reason}</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Evidence list ───────────────────────────────────
|
||||
function EvidenceDisplay({ evidence }) {
|
||||
if (!evidence?.length) return null;
|
||||
const arr = Array.isArray(evidence) ? evidence : [evidence];
|
||||
|
||||
const evidenceLabels = {
|
||||
direct_observation: "👁 Direct Observation",
|
||||
reported_statement: "🗣 Reported Statement",
|
||||
interpretation: "💡 Interpretation",
|
||||
assumption: "❓ Assumption",
|
||||
inferred_relationship: "🔗 Inferred Relationship",
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="mb-4 rounded-lg border border-gray-200 bg-white p-4">
|
||||
<h3 className="mb-2 text-sm font-semibold text-gray-600">
|
||||
Supporting Evidence ({arr.length})
|
||||
</h3>
|
||||
<ul className="space-y-2">
|
||||
{arr.map((item, idx) => (
|
||||
<li
|
||||
key={item.id || `${idx}`}
|
||||
className="rounded border border-gray-200 bg-white px-3 py-2 text-sm leading-relaxed"
|
||||
>
|
||||
<div className="flex items-center gap-2 mb-0.5 flex-wrap">
|
||||
{item.id && (
|
||||
<span className="font-mono text-xs text-gray-400">
|
||||
#{item.id}
|
||||
</span>
|
||||
)}
|
||||
<span
|
||||
className={`inline-block rounded px-1.5 py-0.5 text-[10px] font-medium ${importanceColors[item.importance] || "text-gray-600 bg-gray-100"}`}
|
||||
>
|
||||
{importanceLabels[item.importance]}
|
||||
</span>
|
||||
<span className="inline-block rounded px-1.5 py-0.5 text-[10px] font-medium bg-gray-100 text-gray-700">
|
||||
{evidenceLabels[item.evidenceType] || item.evidenceType}
|
||||
</span>
|
||||
{item.confidence && <ConfidenceBadge level={item.confidence} />}
|
||||
</div>
|
||||
<p className="text-sm">{item.description}</p>
|
||||
{(item.source || item.attribution) && (
|
||||
<p className="mt-0.5 text-xs text-gray-400">
|
||||
Source: {item.source || item.attribution}
|
||||
</p>
|
||||
)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Main component ──────────────────────────────────
|
||||
export default function ReconstructionView({ reconstruction, partial }) {
|
||||
// Handle both v0.2 direct object and wrapped result formats
|
||||
const data = reconstruction;
|
||||
|
||||
if (partial) {
|
||||
return (
|
||||
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-3 text-sm text-yellow-800">
|
||||
⚠ Partial result — some fields failed validation. Showing what was accepted.
|
||||
⚠ Partial result — some fields failed validation. Showing what was
|
||||
accepted.
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const categories = Object.entries(categoryLabels).map(([key, label]) => ({
|
||||
key,
|
||||
label,
|
||||
items: reconstruction[key],
|
||||
}));
|
||||
|
||||
return (
|
||||
<div className="space-y-1">
|
||||
<h2 className="mb-3 text-lg font-semibold">Reconstruction</h2>
|
||||
{categories.map(({ key, label, items }) => (
|
||||
<div key={key} className="mb-4 rounded border border-gray-200 bg-white p-4">
|
||||
<h3 className="mb-2 text-sm font-medium text-gray-600">{label}</h3>
|
||||
<ItemList items={items} />
|
||||
<div className="space-y-4">
|
||||
{/* Classification first */}
|
||||
{data.inputClassification && (
|
||||
<ClassificationDisplay classification={data.inputClassification} />
|
||||
)}
|
||||
|
||||
{/* Summary */}
|
||||
{data.reconstruction?.summary && (
|
||||
<SummaryDisplay reconstruction={data.reconstruction} />
|
||||
)}
|
||||
|
||||
{/* Key differences */}
|
||||
{data.reconstruction?.differences && (
|
||||
<ItemList
|
||||
title="Key Differences"
|
||||
items={data.reconstruction.differences}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Unexplained transitions */}
|
||||
{data.reconstruction?.unexplainedTransitions &&
|
||||
data.reconstruction.unexplainedTransitions.length > 0 && (
|
||||
<ItemList
|
||||
title="Unexplained Transitions"
|
||||
items={data.reconstruction.unexplainedTransitions}
|
||||
renderExtra={(i) =>
|
||||
i.entity && (
|
||||
<p className="mt-1 text-xs text-gray-500">Entity: {i.entity}</p>
|
||||
)
|
||||
}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Contradictions */}
|
||||
{data.reconstruction?.contradictions &&
|
||||
data.reconstruction.contradictions.length > 0 && (
|
||||
<ItemList
|
||||
title="Contradictions"
|
||||
items={data.reconstruction.contradictions}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Important unknowns */}
|
||||
{data.reconstruction?.importantUnknowns &&
|
||||
data.reconstruction.importantUnknowns.length > 0 && (
|
||||
<ItemList
|
||||
title="Important Unknowns"
|
||||
items={data.reconstruction.importantUnknowns}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Plausible interpretations */}
|
||||
{data.reconstruction?.plausibleInterpretations &&
|
||||
data.reconstruction.plausibleInterpretations.length > 0 && (
|
||||
<InterpretationsDisplay
|
||||
interpretations={data.reconstruction.plausibleInterpretations}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Secondary reconstruction categories (actors, systems, etc.) */}
|
||||
{data.reconstruction?.actors && data.reconstruction.actors.length > 0 && (
|
||||
<ItemList title="Actors" items={data.reconstruction.actors} />
|
||||
)}
|
||||
{data.reconstruction?.systemsOrObjects &&
|
||||
data.reconstruction.systemsOrObjects.length > 0 && (
|
||||
<ItemList
|
||||
title="Systems / Objects"
|
||||
items={data.reconstruction.systemsOrObjects}
|
||||
/>
|
||||
)}
|
||||
{data.reconstruction?.expectedStates &&
|
||||
data.reconstruction.expectedStates.length > 0 && (
|
||||
<ItemList
|
||||
title="Expected States"
|
||||
items={data.reconstruction.expectedStates}
|
||||
/>
|
||||
)}
|
||||
{data.reconstruction?.observedStates &&
|
||||
data.reconstruction.observedStates.length > 0 && (
|
||||
<ItemList
|
||||
title="Observed States"
|
||||
items={data.reconstruction.observedStates}
|
||||
/>
|
||||
)}
|
||||
{data.reconstruction?.knownTransitions &&
|
||||
data.reconstruction.knownTransitions.length > 0 && (
|
||||
<ItemList
|
||||
title="Known Transitions"
|
||||
items={data.reconstruction.knownTransitions}
|
||||
renderExtra={(i) => (
|
||||
<div className="mt-1 text-xs text-gray-500">
|
||||
{i.entity && <span>Entity: {i.entity} · </span>}
|
||||
From “{i.previousState}” → To “{i.currentState}” ("{i.explanationStatus}")
|
||||
</div>
|
||||
))}
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Next question — prominent */}
|
||||
<NextQuestionDisplay question={data.nextQuestion} />
|
||||
|
||||
{/* Evidence */}
|
||||
{data.evidence && <EvidenceDisplay evidence={data.evidence} />}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ const MAX_LENGTH = 10000;
|
||||
|
||||
export default function ScenarioForm() {
|
||||
const [scenario, setScenario] = useState("");
|
||||
const [status, setStatus] = useState("idle"); // idle | loading | error | success
|
||||
const [status, setStatus] = useState("idle"); // idle | loading | error | success | partial
|
||||
const [result, setResult] = useState(null);
|
||||
const textareaRef = useRef(null);
|
||||
|
||||
@@ -29,6 +29,10 @@ export default function ScenarioForm() {
|
||||
if (res.ok && data.validationStatus === "valid") {
|
||||
setStatus("success");
|
||||
setResult(data);
|
||||
} else if (data.success) {
|
||||
// Success in analysis but validation may be partial
|
||||
setStatus("success");
|
||||
setResult(data);
|
||||
} else {
|
||||
setStatus("error");
|
||||
setResult(data);
|
||||
@@ -39,8 +43,13 @@ export default function ScenarioForm() {
|
||||
}
|
||||
};
|
||||
|
||||
// Always show diagnostics when there's a result (even if validation failed)
|
||||
const hasDiagnostics = result && (result.reconstruction || result.modelName || result.responseDurationMs !== undefined);
|
||||
// Determine if we have meaningful content to display
|
||||
const hasClassification = result?.inputClassification;
|
||||
const hasReconstruction = result?.reconstruction;
|
||||
const hasNextQuestion = result?.nextQuestion;
|
||||
const hasEvidence = result?.evidence && result.evidence.length > 0;
|
||||
const hasMeaningfulContent =
|
||||
hasClassification || hasReconstruction || hasNextQuestion || hasEvidence;
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
@@ -54,7 +63,9 @@ export default function ScenarioForm() {
|
||||
className="w-full rounded-lg border border-gray-300 px-4 py-3 text-sm focus:border-gray-500 focus:outline-none focus:ring-2 focus:ring-gray-400"
|
||||
/>
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs text-gray-400">{scenario.length}/{MAX_LENGTH}</span>
|
||||
<span className="text-xs text-gray-400">
|
||||
{scenario.length}/{MAX_LENGTH}
|
||||
</span>
|
||||
<button
|
||||
type="submit"
|
||||
disabled={status === "loading" || !scenario.trim()}
|
||||
@@ -65,6 +76,7 @@ export default function ScenarioForm() {
|
||||
</div>
|
||||
</form>
|
||||
|
||||
{/* Error state */}
|
||||
{status === "error" && (
|
||||
<div className="space-y-3">
|
||||
{result?.error && (
|
||||
@@ -72,35 +84,51 @@ export default function ScenarioForm() {
|
||||
Error: {result.error}
|
||||
</div>
|
||||
)}
|
||||
{hasDiagnostics && result?.modelName && (
|
||||
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1.5 text-sm">
|
||||
<dt className="text-gray-500">Model</dt>
|
||||
<dd>{result.modelName}</dd>
|
||||
<dt className="text-gray-500">Duration</dt>
|
||||
<dd>{result.responseDurationMs != null ? `${result.responseDurationMs}ms` : "?"}</dd>
|
||||
</dl>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{status === "success" && result?.reconstruction && (
|
||||
<div className="space-y-4">
|
||||
<ReconstructionView reconstruction={result.reconstruction} />
|
||||
<DiagnosticsView result={result} />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{status === "error" && result?.reconstruction && (
|
||||
<div className="space-y-3">
|
||||
{/* Show partial content even on validation failure */}
|
||||
{(hasClassification || hasReconstruction) && (
|
||||
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
|
||||
⚠ Partial result — some fields failed validation. Showing what was accepted.
|
||||
⚠ Partial result — some fields failed validation. Showing what was
|
||||
accepted.
|
||||
</div>
|
||||
<ReconstructionView reconstruction={result.reconstruction} partial />
|
||||
)}
|
||||
{hasReconstruction && (
|
||||
<ReconstructionView reconstruction={result} partial />
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Success state */}
|
||||
{status === "success" && hasMeaningfulContent && (
|
||||
<div className="space-y-4">
|
||||
<ReconstructionView reconstruction={result} />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Always show diagnostics when we have any result */}
|
||||
{(hasClassification || hasReconstruction || hasNextQuestion) && (
|
||||
<DiagnosticsView result={result} />
|
||||
)}
|
||||
|
||||
{status === "loading" && (
|
||||
<div className="py-12 text-center text-sm text-gray-400">Waiting for model response...</div>
|
||||
<div className="py-12 text-center text-sm text-gray-400">
|
||||
Waiting for model response...
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Empty state */}
|
||||
{status === "idle" && (
|
||||
<div className="rounded-lg border border-dashed border-gray-300 bg-gray-50 px-6 py-8 text-center">
|
||||
<p className="text-sm text-gray-400">
|
||||
Enter a scenario above and click Analyse to begin.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Invalid result with no partial data */}
|
||||
{status === "error" && !result?.error && !hasMeaningfulContent && (
|
||||
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
|
||||
Validation failed — no structured output was produced.
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
/**
|
||||
* Core analysis pipeline — shared by API routes and evaluation harness.
|
||||
* Calls the provider, parses output, validates against Zod schemas (v0.2 first, v0.1 fallback).
|
||||
*/
|
||||
|
||||
import { getConfig } from "../lib/config.js";
|
||||
import { getProvider } from "../lib/llm/provider.js";
|
||||
import { buildPrompt, PROMPT_VERSIONS } from "../lib/reconstruction/prompt.js";
|
||||
import {
|
||||
reconstructionV2Schema,
|
||||
reconstructionSchema as reconstructionV1Schema,
|
||||
} from "../lib/reconstruction/schema.js";
|
||||
|
||||
const MAX_SCENARIO_LENGTH = 10000;
|
||||
const DEFAULT_PROMPT_VERSION = "v0.2";
|
||||
|
||||
/**
|
||||
* Analyse a scenario string through the full pipeline.
|
||||
* @param {string} scenario - The scenario text to analyse
|
||||
* @param {object} [opts]
|
||||
* @param {"v0.1" | "v0.2"} [opts.promptVersion="v0.2"] - Prompt version to use
|
||||
* @returns {Promise<object>} Analysis result with diagnostics
|
||||
*/
|
||||
export async function analyseScenario(scenario, opts = {}) {
|
||||
const startTime = Date.now();
|
||||
|
||||
// ── Input validation ───────────────────────────────
|
||||
if (typeof scenario !== "string") {
|
||||
return buildErrorResponse("Input must be a string", startTime);
|
||||
}
|
||||
|
||||
const trimmed = scenario.trim();
|
||||
if (trimmed.length === 0) {
|
||||
return buildErrorResponse("Scenario cannot be empty", startTime);
|
||||
}
|
||||
if (trimmed.length > MAX_SCENARIO_LENGTH) {
|
||||
return buildErrorResponse(`Scenario must be under ${MAX_SCENARIO_LENGTH} characters`, startTime);
|
||||
}
|
||||
|
||||
// ── Configuration check ────────────────────────────
|
||||
const configResult = getConfig();
|
||||
if (!configResult.ok) {
|
||||
return buildErrorResponse("Invalid server configuration", startTime, "500");
|
||||
}
|
||||
|
||||
const { OLLAMA_BASE_URL: _ignored, OLLAMA_MODEL } = configResult.config;
|
||||
const promptVersion = opts.promptVersion || DEFAULT_PROMPT_VERSION;
|
||||
|
||||
// ── Build prompt ───────────────────────────────────
|
||||
let promptObj;
|
||||
try {
|
||||
promptObj = await buildPrompt(trimmed, promptVersion);
|
||||
} catch (e) {
|
||||
return buildErrorResponse(`Failed to build prompt: ${e.message}`, startTime);
|
||||
}
|
||||
|
||||
// ── Call provider ──────────────────────────────────
|
||||
const provider = getProvider();
|
||||
let rawResponse;
|
||||
try {
|
||||
rawResponse = await provider.generateReconstruction(promptObj.prompt, OLLAMA_MODEL);
|
||||
} catch (e) {
|
||||
return buildErrorResponse(
|
||||
e.message || "Provider error during analysis",
|
||||
Date.now() - startTime
|
||||
);
|
||||
}
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
// Try to capture raw response for diagnostics
|
||||
let rawResponseStr;
|
||||
try {
|
||||
rawResponseStr = JSON.stringify(rawResponse);
|
||||
} catch {
|
||||
rawResponseStr = String(rawResponse).slice(0, 2000);
|
||||
}
|
||||
|
||||
// ── Validate against v0.2 schema (preferred) ──────
|
||||
const resultV2 = tryValidateAgainstSchema(rawResponse, reconstructionV2Schema);
|
||||
if (resultV2.valid) {
|
||||
return buildSuccessResultV2(resultV2.data, OLLAMA_MODEL, duration, promptVersion);
|
||||
}
|
||||
|
||||
// ── Fallback to v0.1 schema ────────────────────────
|
||||
const resultV1 = tryValidateAgainstSchema(rawResponse, reconstructionV1Schema);
|
||||
if (resultV1.valid) {
|
||||
return buildSuccessResultV1(resultV1.data, OLLAMA_MODEL, duration, promptVersion);
|
||||
}
|
||||
|
||||
// ── Neither schema matched — partial failure ───────
|
||||
return buildPartialResult(
|
||||
rawResponseStr?.slice(0, 2000),
|
||||
resultV2.error ?? resultV1.error,
|
||||
OLLAMA_MODEL,
|
||||
duration,
|
||||
promptVersion
|
||||
);
|
||||
}
|
||||
|
||||
/** Attempt validation against a Zod schema */
|
||||
function tryValidateAgainstSchema(data, schema) {
|
||||
if (!schema.safeParse) {
|
||||
return { valid: false, error: new Error("Schema does not support safeParse") };
|
||||
}
|
||||
const result = schema.safeParse(data);
|
||||
return result.success ? { valid: true, data: result.data } : { valid: false, error: result.error };
|
||||
}
|
||||
|
||||
// ── Result builders ──────────────────────────────────
|
||||
|
||||
function buildErrorResponse(message, elapsed, statusCode = 500) {
|
||||
return {
|
||||
success: false,
|
||||
error: message,
|
||||
modelName: null,
|
||||
responseDurationMs: elapsed,
|
||||
validationStatus: "invalid",
|
||||
rawResponse: null,
|
||||
promptVersion: null,
|
||||
statusCode,
|
||||
};
|
||||
}
|
||||
|
||||
function buildSuccessResultV2(data, model, duration, version) {
|
||||
return {
|
||||
success: true,
|
||||
validationStatus: "valid",
|
||||
modelName: model,
|
||||
responseDurationMs: duration,
|
||||
rawResponse: JSON.stringify(data).slice(0, 3000),
|
||||
promptVersion: version,
|
||||
inputClassification: data.inputClassification,
|
||||
reconstruction: data.reconstruction,
|
||||
evidence: data.evidence,
|
||||
nextQuestion: data.nextQuestion,
|
||||
errors: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function buildSuccessResultV1(data, model, duration, version) {
|
||||
return {
|
||||
success: true,
|
||||
validationStatus: "valid",
|
||||
modelName: model,
|
||||
responseDurationMs: duration,
|
||||
rawResponse: JSON.stringify(data).slice(0, 3000),
|
||||
promptVersion: version,
|
||||
inputClassification: null,
|
||||
reconstruction: data,
|
||||
evidence: undefined,
|
||||
nextQuestion: undefined,
|
||||
errors: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function buildPartialResult(rawResp, error, model, duration, version) {
|
||||
let errors = [];
|
||||
if (error && typeof error.flatten === "function") {
|
||||
errors = error.flatten().fieldErrors
|
||||
? Object.entries(error.flatten().fieldErrors).flatMap(([k, v]) => [`${k}: ${v.join(", ")}`])
|
||||
: [String(error)];
|
||||
} else if (error) {
|
||||
errors = [String(error).slice(0, 500)];
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
validationStatus: "invalid",
|
||||
modelName: model,
|
||||
responseDurationMs: duration,
|
||||
rawResponse: rawResp?.slice(0, 2000),
|
||||
promptVersion: version,
|
||||
inputClassification: null,
|
||||
reconstruction: null,
|
||||
evidence: undefined,
|
||||
nextQuestion: undefined,
|
||||
errors,
|
||||
};
|
||||
}
|
||||
|
||||
export { PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION };
|
||||
+3
-5
@@ -93,11 +93,9 @@ async function detectChatSupport(baseUrl) {
|
||||
|
||||
class OllamaLlmProvider {
|
||||
async generateReconstruction(scenario, modelName) {
|
||||
const { buildPrompt } = await import("@/lib/reconstruction/prompt");
|
||||
|
||||
let rawPrompt = buildPrompt(scenario);
|
||||
// Stronger JSON hint since we can't use format:json on older Ollama
|
||||
const prompt = rawPrompt + `\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.`;
|
||||
// scenario is ALREADY a fully-built prompt text (built by analyseScenario).
|
||||
// Do NOT call buildPrompt() again — that would double-wrap the prompt.
|
||||
const prompt = scenario;
|
||||
|
||||
const baseUrl = process.env.OLLAMA_BASE_URL;
|
||||
if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");
|
||||
|
||||
@@ -1,4 +1,16 @@
|
||||
export function buildPrompt(scenario) {
|
||||
import { promises as fs } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { dirname, join } from "node:path";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
const PROMPTS_DIR = join(__dirname, "../../prompts");
|
||||
|
||||
/** Available prompt versions */
|
||||
export const PROMPT_VERSIONS = ["v0.1", "v0.2"];
|
||||
|
||||
/** Build a v0.1 (extraction-only) prompt inline for backward compatibility */
|
||||
function buildV1Prompt(scenario) {
|
||||
return `You are a neutral analyst performing an evidence-based reconstruction of the following scenario.
|
||||
|
||||
Rules:
|
||||
@@ -29,3 +41,38 @@ Return valid JSON matching this structure exactly:
|
||||
|
||||
Return ONLY the JSON object. No markdown, no explanation, no preamble.`;
|
||||
}
|
||||
|
||||
/** Load a versioned prompt from disk and substitute {{SCENARIO}} */
|
||||
async function buildV2Prompt(scenario) {
|
||||
try {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.2.md"),
|
||||
"utf-8",
|
||||
);
|
||||
return content.replace("{{SCENARIO}}", scenario);
|
||||
} catch {
|
||||
// Fall back to v0.1 prompt if v0.2 file is missing
|
||||
return buildV1Prompt(scenario);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build an analysis prompt for the given version.
|
||||
* @param {"v0.1" | "v0.2"} [version="v0.2"]
|
||||
* @returns {Promise<{prompt: string, version: string}>}
|
||||
*/
|
||||
export async function buildPrompt(scenario, version = "v0.2") {
|
||||
let prompt;
|
||||
switch (version) {
|
||||
case "v0.1":
|
||||
prompt = buildV1Prompt(scenario);
|
||||
break;
|
||||
default: // v0.2
|
||||
prompt = await buildV2Prompt(scenario);
|
||||
break;
|
||||
}
|
||||
|
||||
const strongJsonHint =
|
||||
"\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.";
|
||||
return { prompt: prompt + strongJsonHint, version };
|
||||
}
|
||||
|
||||
+183
-16
@@ -1,38 +1,59 @@
|
||||
import { z } from "zod";
|
||||
|
||||
const confidenceEnum = z.enum(["low", "medium", "high"]);
|
||||
// ──────────────────────────────────────────────
|
||||
// Shared enums (v0.1 & v0.2)
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
const itemSchema = z.object({
|
||||
export const confidenceEnum = z.enum(["low", "medium", "high"]);
|
||||
const importanceEnum = z.enum([
|
||||
"incidental",
|
||||
"supporting",
|
||||
"important",
|
||||
"critical",
|
||||
]);
|
||||
const expectedInfoValueEnum = z.enum(["low", "medium", "high"]);
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// v0.1 — extraction-only schema (preserved)
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
const confidenceEnumV1 = z.enum(["low", "medium", "high"]);
|
||||
|
||||
const itemSchemaV1 = z.object({
|
||||
id: z.string().min(1),
|
||||
description: z.string().min(1),
|
||||
confidence: confidenceEnum,
|
||||
confidence: confidenceEnumV1,
|
||||
});
|
||||
|
||||
export const reconstructionSchema = z.object({
|
||||
observations: z.array(itemSchema),
|
||||
observations: z.array(itemSchemaV1),
|
||||
reportedClaims: z.array(
|
||||
itemSchema.extend({
|
||||
attributedTo: z.union([z.string().min(1), z.null()]).optional().nullable(),
|
||||
})
|
||||
itemSchemaV1.extend({
|
||||
attributedTo: z
|
||||
.union([z.string().min(1), z.null()])
|
||||
.optional()
|
||||
.nullable(),
|
||||
}),
|
||||
),
|
||||
assumptions: z.array(itemSchema),
|
||||
entities: z.array(itemSchema),
|
||||
assumptions: z.array(itemSchemaV1),
|
||||
entities: z.array(itemSchemaV1),
|
||||
transitions: z.array(
|
||||
itemSchema.extend({
|
||||
itemSchemaV1.extend({
|
||||
entity: z.string().min(1),
|
||||
previousState: z.string().min(1),
|
||||
currentState: z.string().min(1),
|
||||
explanationStatus: z.string().min(1),
|
||||
})
|
||||
}),
|
||||
),
|
||||
expectedButMissing: z.array(itemSchema),
|
||||
presentButUnexpected: z.array(itemSchema),
|
||||
contradictions: z.array(itemSchema),
|
||||
openUncertainties: z.array(itemSchema),
|
||||
expectedButMissing: z.array(itemSchemaV1),
|
||||
presentButUnexpected: z.array(itemSchemaV1),
|
||||
contradictions: z.array(itemSchemaV1),
|
||||
openUncertainties: z.array(itemSchemaV1),
|
||||
});
|
||||
|
||||
// v0.1 analyse response (used internally)
|
||||
export const analyseResponseSchema = z.object({
|
||||
reconstruction: reconstructionSchema,
|
||||
reconstruction: z.union([reconstructionSchema, z.null()]),
|
||||
modelName: z.string(),
|
||||
responseDurationMs: z.number(),
|
||||
validationStatus: z.enum(["valid", "partial", "invalid"]),
|
||||
@@ -48,6 +69,141 @@ export const healthResponseSchema = z.object({
|
||||
error: z.string().nullable(),
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// v0.2 — reasoning classification + reconstruction
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
export const inputTypes =
|
||||
/** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").INPUT_TYPE_VALUE>} */ (
|
||||
z.enum([
|
||||
"observed_problem",
|
||||
"unexplained_change",
|
||||
"contradiction",
|
||||
"decision_request",
|
||||
"causal_claim",
|
||||
"reported_claim",
|
||||
"fault_report",
|
||||
"ambiguous_statement",
|
||||
"question",
|
||||
"desired_outcome",
|
||||
"insufficient_context",
|
||||
"other",
|
||||
])
|
||||
);
|
||||
|
||||
export const reasoningModes =
|
||||
/** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").REASONING_MODE_VALUE>} */ (
|
||||
z.enum([
|
||||
"establish_baseline",
|
||||
"identify_difference",
|
||||
"reconstruct_transition",
|
||||
"decompose_aggregate",
|
||||
"validate_measurement",
|
||||
"validate_claim",
|
||||
"investigate_contradiction",
|
||||
"clarify_meaning",
|
||||
"decision_support",
|
||||
"fault_investigation",
|
||||
"identify_missing_information",
|
||||
"test_possible_explanations",
|
||||
"other",
|
||||
])
|
||||
);
|
||||
|
||||
const evidenceRecordSchema = z.object({
|
||||
id: z.string().min(1),
|
||||
description: z.string().min(1),
|
||||
evidenceType: z.enum([
|
||||
"direct_observation",
|
||||
"reported_statement",
|
||||
"interpretation",
|
||||
"assumption",
|
||||
"inferred_relationship",
|
||||
]),
|
||||
source: z.string().optional(),
|
||||
attribution: z.string().nullable().optional(),
|
||||
confidence: confidenceEnum,
|
||||
importance: importanceEnum,
|
||||
});
|
||||
|
||||
const reconstructionSchemaV2 = z.object({
|
||||
summary: z.string().min(1),
|
||||
actors: z.array(itemSchemaV1),
|
||||
systemsOrObjects: z.array(itemSchemaV1),
|
||||
expectedStates: z.array(itemSchemaV1),
|
||||
observedStates: z.array(itemSchemaV1),
|
||||
differences: z.array(itemSchemaV1),
|
||||
knownTransitions: z.array(
|
||||
itemSchemaV1.extend({
|
||||
entity: z.string().min(1),
|
||||
previousState: z.string().min(1),
|
||||
currentState: z.string().min(1),
|
||||
explanationStatus: z.string().min(1),
|
||||
}),
|
||||
),
|
||||
unexplainedTransitions: z.array(
|
||||
itemSchemaV1.extend({
|
||||
entity: z.string().min(1).optional(),
|
||||
previousState: z.string().min(1).optional(),
|
||||
currentState: z.string().min(1).optional(),
|
||||
}),
|
||||
),
|
||||
contradictions: z.array(itemSchemaV1),
|
||||
importantUnknowns: z.array(itemSchemaV1),
|
||||
plausibleInterpretations: z.array(
|
||||
z.object({
|
||||
id: z.string().min(1),
|
||||
description: z.string().min(1),
|
||||
supportingEvidenceIds: z.array(z.string()),
|
||||
assumptionsRequired: z.array(z.string()).optional().default([]),
|
||||
confidence: confidenceEnum,
|
||||
}),
|
||||
),
|
||||
});
|
||||
|
||||
const inputClassificationSchema = z.object({
|
||||
primaryType: inputTypes,
|
||||
secondaryTypes: z.array(inputTypes).optional().default([]),
|
||||
reasoningModes: z.array(reasoningModes).optional().default([]),
|
||||
classificationReason: z.string().min(1),
|
||||
confidence: confidenceEnum,
|
||||
});
|
||||
|
||||
const nextQuestionSchema = z.object({
|
||||
id: z.string().min(1),
|
||||
question: z.string().min(1),
|
||||
targets: z.array(z.string()),
|
||||
reason: z.string().min(1),
|
||||
expectedInformationValue: expectedInfoValueEnum,
|
||||
reasoningMode: reasoningModes.optional().default("other"),
|
||||
});
|
||||
|
||||
// v0.2 complete analysis response (what the model produces)
|
||||
export const reconstructionV2Schema = z.object({
|
||||
inputClassification: inputClassificationSchema,
|
||||
reconstruction: reconstructionSchemaV2,
|
||||
evidence: z.array(evidenceRecordSchema),
|
||||
nextQuestion: nextQuestionSchema,
|
||||
});
|
||||
|
||||
// Outer wrapper for API return (includes diagnostics + v0.2 data)
|
||||
export const analyseResponseV2Schema = z.object({
|
||||
inputClassification: inputClassificationSchema.optional(),
|
||||
reconstruction: reconstructionSchemaV2.optional().nullable(),
|
||||
evidence: z.array(evidenceRecordSchema).optional(),
|
||||
nextQuestion: nextQuestionSchema.optional(),
|
||||
modelName: z.string(),
|
||||
responseDurationMs: z.number(),
|
||||
validationStatus: z.enum(["valid", "partial", "invalid"]),
|
||||
rawResponse: z.string().optional(),
|
||||
errors: z.array(z.string()).optional(),
|
||||
promptVersion: z.string().optional(),
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Parsing helpers
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
export function parseReconstruction(raw) {
|
||||
if (typeof raw === "string") {
|
||||
try {
|
||||
@@ -58,3 +214,14 @@ export function parseReconstruction(raw) {
|
||||
}
|
||||
return reconstructionSchema.parse(raw);
|
||||
}
|
||||
|
||||
export function parseReconstructionV2(raw) {
|
||||
if (typeof raw === "string") {
|
||||
try {
|
||||
raw = JSON.parse(raw);
|
||||
} catch {
|
||||
throw new SyntaxError("Model response is not valid JSON");
|
||||
}
|
||||
}
|
||||
return reconstructionV2Schema.parse(raw);
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"type": "module",
|
||||
"name": "confidence-engine",
|
||||
"version": "0.1.0",
|
||||
"version": "0.2.0-experimental",
|
||||
"private": true,
|
||||
"description": "Experimental prototype for evidence-based situation reconstruction using local LLMs",
|
||||
"scripts": {
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
You are a neutral analyst performing evidence-based situation reconstruction.
|
||||
|
||||
## Rules
|
||||
|
||||
1. Do NOT invent facts, context or causes. Only include information present in the scenario or clearly implied.
|
||||
2. First determine what kind of input has been supplied. Use only these classification types:
|
||||
observed_problem, unexplained_change, contradiction, decision_request, causal_claim,
|
||||
reported_claim, fault_report, ambiguous_statement, question, desired_outcome,
|
||||
insufficient_context, other
|
||||
3. Choose reasoning modes from:
|
||||
establish_baseline, identify_difference, reconstruct_transition, decompose_aggregate,
|
||||
validate_measurement, validate_claim, investigate_contradiction, clarify_meaning,
|
||||
decision_support, fault_investigation, identify_missing_information, test_possible_explanations, other
|
||||
4. Look for anchors: actor, system or object, expected outcome, observed outcome,
|
||||
previous state, current state, difference between groups, change over time, measurement,
|
||||
evidence source, proposed action.
|
||||
5. Identify meaningful differences (e.g., some succeed while others fail; revenue rises while cash falls).
|
||||
6. Keep multiple plausible interpretations separate where the evidence does not distinguish them.
|
||||
7. Distinguish: what was said / what it may mean / why it may have been said.
|
||||
8. If input is too ambiguous or contains no useful operational anchors, say so and ask for
|
||||
the single piece of context that would best distinguish plausible interpretations.
|
||||
|
||||
## Confidence scale
|
||||
|
||||
- low — weak evidence, speculation, or missing information
|
||||
- medium — reasonable inference from available evidence
|
||||
- high — strong evidence, direct observation, or confirmed fact
|
||||
|
||||
## Importance scale (evidence records)
|
||||
|
||||
- incidental — minor detail, unlikely to affect conclusions
|
||||
- supporting — adds context but not critical
|
||||
- important — materially affects understanding of the situation
|
||||
- critical — essential to resolving the situation; without it conclusions cannot be drawn
|
||||
|
||||
## Expected information value (next question)
|
||||
|
||||
- low — marginally useful even if answered
|
||||
- medium — meaningfully clarifies the situation
|
||||
- high — would significantly distinguish between plausible explanations or fill a gap in understanding
|
||||
|
||||
## Next question selection criteria
|
||||
|
||||
Prefer questions that:
|
||||
- clarify a major difference
|
||||
- establish a baseline
|
||||
- explain an important transition
|
||||
- test an unsupported claim
|
||||
- distinguish between plausible explanations
|
||||
- request measurable evidence
|
||||
- identify who or what is affected
|
||||
- establish timing
|
||||
|
||||
Avoid questions that:
|
||||
- have already been answered
|
||||
- assume a cause
|
||||
- jump to a solution
|
||||
- ask about motive before the observable situation is understood
|
||||
- focus on incidental wording
|
||||
- are too broad to produce useful information
|
||||
- combine many unrelated questions
|
||||
|
||||
## Output format — return this exact JSON structure
|
||||
|
||||
Return a JSON object with exactly these four top-level keys (use **camelCase**):
|
||||
|
||||
```json
|
||||
{
|
||||
"inputClassification": {
|
||||
"primaryType": "<one of: observed_problem, unexplained_change, contradiction, decision_request, causal_claim, reported_claim, fault_report, ambiguous_statement, question, desired_outcome, insufficient_context, other>",
|
||||
"secondaryTypes": ["<optional additional types from the same list>"],
|
||||
"reasoningModes": ["<one or more of: establish_baseline, identify_difference, reconstruct_transition, decompose_aggregate, validate_measurement, validate_claim, investigate_contradiction, clarify_meaning, decision_support, fault_investigation, identify_missing_information, test_possible_explanations, other>"],
|
||||
"classificationReason": "<brief explanation of why you chose the primary type>",
|
||||
"confidence": "<low | medium | high>"
|
||||
},
|
||||
"reconstruction": {
|
||||
"summary": "<one-sentence overview of the situation>",
|
||||
"actors": [{"id": "<any unique string>", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"systemsOrObjects": [{"id": "<any unique string>", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"expectedStates": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"observedStates": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"differences": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"knownTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}],
|
||||
"unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "..."}],
|
||||
"contradictions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"importantUnknowns": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": ["<ids that support this interpretation>"], "assumptionsRequired": [], "confidence": "<low|medium|high>"}]
|
||||
},
|
||||
"evidence": [
|
||||
{
|
||||
"id": "<any unique string>",
|
||||
"description": "...",
|
||||
"evidenceType": "<direct_observation | reported_statement | interpretation | assumption | inferred_relationship>",
|
||||
"source": "<optional — who/where this came from>",
|
||||
"attribution": null,
|
||||
"confidence": "<low | medium | high>",
|
||||
"importance": "<incidental | supporting | important | critical>"
|
||||
}
|
||||
],
|
||||
"nextQuestion": {
|
||||
"id": "<any unique string>",
|
||||
"question": "<one precise question>",
|
||||
"targets": ["<what this question targets — e.g. 'actor', 'system', 'expectedOutcome'>"],
|
||||
"reason": "<why answering this is important>",
|
||||
"expectedInformationValue": "<low | medium | high>",
|
||||
"reasoningMode": "<optional reasoning mode from the list above>"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
CRITICAL RULES for JSON output:
|
||||
1. Use **exactly** the key names shown above (camelCase, no snake_case).
|
||||
2. The four top-level keys must be: `inputClassification`, `reconstruction`, `evidence`, `nextQuestion`.
|
||||
3. Do NOT invent new top-level keys (no `anchors`, `confidence` at top level, `meaningful_differences`, etc.).
|
||||
4. Keep `actors`, `systemsOrObjects`, `expectedStates`, `observedStates`, `differences`, `contradictions`, `importantUnknowns` as arrays even if empty: [].
|
||||
5. Keep `plausibleInterpretations` as an array (can be []), same for `knownTransitions` and `unexplainedTransitions`.
|
||||
6. Each object in arrays must have at least `id`, `description`, `confidence`.
|
||||
|
||||
Scenario:
|
||||
{{SCENARIO}}
|
||||
|
||||
Return ONLY the JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.
|
||||
Reference in New Issue
Block a user