From d72c7c5465c820220bb1138864c74c744509b118 Mon Sep 17 00:00:00 2001 From: robbond Date: Sat, 1 Aug 2026 14:32:40 +0100 Subject: [PATCH] chore: establish clean v0.2 baseline Include only the working reconstruction prototype with Ollama integration: - double-wrapping fix (lib/llm/provider.js) - explicit v0.2 JSON output schema (prompts/reconstruct-v0.2.md) - Zod validation layer (lib/reconstruction/schema.js) - shared core analysis path (lib/analysis.js) - prompt versioning infrastructure (lib/reconstruction/prompt.js) - provider abstraction - functioning Ollama provider path - updated API route with centralized analysis - UI components displaying v0.2 data and validation errors - .gitignore rules for generated evaluation artifacts Exclude: evaluator experiments, diagnostic tests, debug scripts, generated artifacts, comparison findings, test data tied to evaluator. --- .gitignore | 5 + app/api/analyse/route.js | 108 ++------ components/diagnostics-view.jsx | 47 +++- components/reconstruction-view.jsx | 424 ++++++++++++++++++++++++++--- components/scenario-form.jsx | 72 +++-- lib/analysis.js | 182 +++++++++++++ lib/llm/provider.js | 8 +- lib/reconstruction/prompt.js | 51 +++- lib/reconstruction/schema.js | 199 ++++++++++++-- package.json | 2 +- prompts/reconstruct-v0.2.md | 122 +++++++++ 11 files changed, 1048 insertions(+), 172 deletions(-) create mode 100644 lib/analysis.js create mode 100644 prompts/reconstruct-v0.2.md diff --git a/.gitignore b/.gitignore index a898f55..dd5091d 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,8 @@ Thumbs.db npm-debug.log* yarn-debug.log* yarn-error.log* + +# Generated evaluation artifacts (regenerated each run) +evaluation-results/ +provider-debug-results/ +tests-results/ diff --git a/app/api/analyse/route.js b/app/api/analyse/route.js index 31ac1e1..fb9563c 100644 --- a/app/api/analyse/route.js +++ b/app/api/analyse/route.js @@ -1,101 +1,49 @@ -import { getConfig } from "@/lib/config"; -import { getProvider } from "@/lib/llm/provider"; -import { reconstructionSchema } from "@/lib/reconstruction/schema"; - -const MAX_SCENARIO_LENGTH = 10000; +import { + analyseScenario, + PROMPT_VERSIONS, + DEFAULT_PROMPT_VERSION, +} from "@/lib/analysis"; export async function POST(request) { - const startTime = Date.now(); - let rawResponse = null; - try { const body = await request.json(); - + if (!body.scenario || typeof body.scenario !== "string") { return Response.json( { error: "Request must include a 'scenario' string field" }, - { status: 400 } + { status: 400 }, ); } - const trimmed = body.scenario.trim(); - - if (trimmed.length === 0) { + // Optional prompt version override + let promptVersion = DEFAULT_PROMPT_VERSION; + if (body.promptVersion && PROMPT_VERSIONS.includes(body.promptVersion)) { + promptVersion = body.promptVersion; + } + + const result = await analyseScenario(body.scenario, { promptVersion }); + + if (!result.success) { return Response.json( - { error: "Scenario cannot be empty" }, - { status: 400 } + { ...result, reconstruction: result.reconstruction || null }, + { status: Number(result.statusCode) || 500 }, ); } - if (trimmed.length > MAX_SCENARIO_LENGTH) { - return Response.json( - { error: `Scenario must be under ${MAX_SCENARIO_LENGTH} characters` }, - { status: 400 } - ); - } - - const configResult = getConfig(); - if (!configResult.ok) { - return Response.json( - { error: "Invalid server configuration" }, - { status: 500 } - ); - } - - const { OLLAMA_BASE_URL, OLLAMA_MODEL } = configResult.config; - const provider = getProvider(); - - // Attempt parse to capture raw for debugging - let reconstruction; - try { - reconstruction = await provider.generateReconstruction(trimmed, OLLAMA_MODEL); - } catch (e) { - return Response.json( - { - error: e.message || "Unknown server error", - responseDurationMs: Date.now() - startTime, - modelName: OLLAMA_MODEL, - validationStatus: "invalid", - }, - { status: 500 } - ); - } - - // Try to stringify for rawResponse display (safe even if it's already an object) - try { - rawResponse = JSON.stringify(reconstruction); - } catch { - rawResponse = String(reconstruction).slice(0, 2000); - } - - const duration = Date.now() - startTime; - - // Validate with Zod schema - const validationResult = reconstructionSchema.safeParse(reconstruction); - - if (!validationResult.success) { - return Response.json({ - reconstruction: null, - modelName: OLLAMA_MODEL, - responseDurationMs: duration, - validationStatus: "invalid", - rawResponse: rawResponse?.slice(0, 2000), - errors: validationResult.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`), - }); - } - return Response.json({ - reconstruction: validationResult.data, - modelName: OLLAMA_MODEL, - responseDurationMs: duration, - validationStatus: "valid", - rawResponse: rawResponse?.slice(0, 2000), + inputClassification: result.inputClassification, + reconstruction: result.reconstruction, + evidence: result.evidence, + nextQuestion: result.nextQuestion, + modelName: result.modelName, + responseDurationMs: result.responseDurationMs, + validationStatus: result.validationStatus, + promptVersion: result.promptVersion, }); } catch (e) { - const duration = Date.now() - startTime; return Response.json( - { error: e.message || "Unknown server error", responseDurationMs: duration }, - { status: 500 } + { error: e.message || "Unknown server error", responseDurationMs: 0 }, + { status: 500 }, ); } } diff --git a/components/diagnostics-view.jsx b/components/diagnostics-view.jsx index 535c4e4..60a5d05 100644 --- a/components/diagnostics-view.jsx +++ b/components/diagnostics-view.jsx @@ -10,17 +10,40 @@ const ValidationIndicator = ({ status }) => { invalid: "❌ Validation failed", }; return ( -
+
{labels[status] || status}
); }; +const validationIcons = { + valid: "✅", + partial: "⚠️", + invalid: "❌", +}; + export default function DiagnosticsView({ result }) { + if (!result) return null; + const metrics = [ { label: "Model", value: result.modelName || "?" }, - { label: "Duration", value: result.responseDurationMs != null ? `${result.responseDurationMs}ms` : "?" }, - { label: "Validation", value: }, + { label: "Provider", value: "Ollama" }, + { label: "Prompt version", value: result.promptVersion || "?" }, + { + label: "Duration", + value: + result.responseDurationMs != null + ? `${result.responseDurationMs}ms` + : "?", + }, + { + label: "Validation", + value: ( + + ), + }, ]; return ( @@ -35,16 +58,32 @@ export default function DiagnosticsView({ result }) { ))} + {/* Collapsed raw output for debugging */} {result.rawResponse && (
- View raw model response + View raw model response ( + {(result.rawResponse?.length || 0).toLocaleString()} chars) 
             {result.rawResponse}
)} + + {/* Errors if present */} + {result.errors && result.errors.length > 0 && ( +
+ + Validation errors ({result.errors.length}) + +
    + {result.errors.map((err, i) => ( +
  • {err}
    • + ))} +
+
+ )}
); } diff --git a/components/reconstruction-view.jsx b/components/reconstruction-view.jsx index 67e8034..e740631 100644 --- a/components/reconstruction-view.jsx +++ b/components/reconstruction-view.jsx @@ -1,15 +1,8 @@ -const categoryLabels = { - observations: "Direct Observations", - reportedClaims: "Reported Claims", - assumptions: "Unsupported Assumptions", - entities: "Entities", - transitions: "Transitions", - expectedButMissing: "Expected But Missing", - presentButUnexpected: "Present But Unexpected", - contradictions: "Contradictions", - openUncertainties: "Open Uncertainties", -}; +"use client"; +import { useMemo } from "react"; + +// ── Confidence badge (shared) ──────────────────────── const confidenceColor = { low: "text-red-600 bg-red-50 border-red-200", medium: "text-yellow-700 bg-yellow-50 border-yellow-200", @@ -17,54 +10,401 @@ const confidenceColor = { }; const ConfidenceBadge = ({ level }) => ( - + {level} ); -function ItemList({ items, renderExtra }) { - if (!items?.length) return

None identified

; - +// ── Evidence type labels (shared) ─────────────────── +const evidenceTypeLabels = { + direct_observation: "Direct Observation", + reported_statement: "Reported Statement", + interpretation: "Interpretation", + assumption: "Assumption", + inferred_relationship: "Inferred Relationship", +}; + +const importanceColors = { + incidental: "text-gray-500 bg-gray-50 border-gray-200", + supporting: "text-blue-700 bg-blue-50 border-blue-200", + important: "text-orange-700 bg-orange-50 border-orange-200", + critical: "text-red-800 bg-red-50 border-red-300 font-semibold", +}; + +const importanceLabels = { + incidental: "Incidental", + supporting: "Supporting", + important: "Important", + critical: "Critical", +}; + +// ── Input classification display ──────────────────── +function ClassificationDisplay({ classification }) { + if (!classification) return null; + const p = classification.primaryType || classification.primary_type; + const sec = + classification.secondaryTypes || classification.secondary_types || []; + const modes = + classification.reasoningModes || classification.reasoning_modes || []; + + // Normalize camelCase to snake_case for display if needed + const primaryLabel = String(p) + .replace(/_/g, " ") + .replace(/\b\w/g, (c) => c.toUpperCase()); + const secLabels = sec.map((s) => + s.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()), + ); + const modeLabels = modes.map((m) => + m.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()), + ); + return ( - +
+

+ Input Classification +

+
+
Primary type
+
{primaryLabel}
+ {secLabels.length > 0 && ( + <> +
Secondary types
+
{secLabels.join(" · ")}
+ + )} + {modeLabels.length > 0 && ( + <> +
Reasoning modes
+
{modeLabels.join(" · ")}
+ + )} +
Classification reason
+
+ {classification.classificationReason || + classification.classification_reason} +
+
Confidence
+
+ +
+
+
); } +// ── Reconstruction summary ────────────────────────── +function SummaryDisplay({ reconstruction }) { + if (!reconstruction?.summary) return null; + const summary = reconstruction.summary || reconstruction.Summary; + return ( +
+

+ Reconstruction Summary +

+

{summary}

+
+ ); +} + +// ── Generic item list (used for multiple sections) ── +function ItemList({ title, items, renderExtra }) { + const count = items?.length; + if (!count) return null; // hide empty sections entirely + + const itemsArr = Array.isArray(items) ? items : [items]; + + return ( +
+

+ {title} ({count}) +

+
    + {itemsArr.map((item, idx) => ( +
  • +
    + {item.id && ( + + #{item.id} + + )} + {item.confidence && } + {item.importance && ( + + {importanceLabels[item.importance]} + + )} +
    +

    {item.description}

    + {renderExtra && renderExtra(item)} +
    • + ))} +
+
+ ); +} + +// ── Plausible interpretations ─────────────────────── +function InterpretationsDisplay({ interpretations }) { + if (!interpretations?.length) return null; + const arr = Array.isArray(interpretations) + ? interpretations + : [interpretations]; + + return ( +
+

+ Plausible Interpretations ({arr.length}) +

+
    + {arr.map((interp, idx) => ( +
  • +
    + + {interp.description} + + {interp.confidence && ( + + )} +
    + {interp.supportingEvidenceIds?.length > 0 && ( +

    + Supporting evidence: {interp.supportingEvidenceIds.join(", ")} +

    + )} + {interp.assumptionsRequired?.length > 0 && ( +

    + Requires assumptions: {interp.assumptionsRequired.join("; ")} +

    + )} +
    • + ))} +
+
+ ); +} + +// ── Next question (prominent) ─────────────────────── +function NextQuestionDisplay({ question }) { + if (!question?.question) return null; + const q = question.question || question.Question; + const targets = question.targets || question.Targets || []; + const reason = question.reason || question.Reason || ""; + const value = + question.expectedInformationValue || + question.expected_information_value || + "medium"; + + const valueLabel = + { low: "Low", medium: "Medium", high: "High" }[value] || "Medium"; + const valueColor = + { + low: "bg-yellow-100 text-yellow-800", + medium: "bg-blue-100 text-blue-800", + high: "bg-green-100 text-green-800", + }[value] || ""; + + return ( +
+
+

Next Question

+ + {valueLabel} value + +
+

{q}

+ {targets.length > 0 && ( +

Targets: {targets.join(", ")}

+ )} + {reason && ( +

Because: {reason}

+ )} +
+ ); +} + +// ── Evidence list ─────────────────────────────────── +function EvidenceDisplay({ evidence }) { + if (!evidence?.length) return null; + const arr = Array.isArray(evidence) ? evidence : [evidence]; + + const evidenceLabels = { + direct_observation: "👁 Direct Observation", + reported_statement: "🗣 Reported Statement", + interpretation: "💡 Interpretation", + assumption: "❓ Assumption", + inferred_relationship: "🔗 Inferred Relationship", + }; + + return ( +
+

+ Supporting Evidence ({arr.length}) +

+
    + {arr.map((item, idx) => ( +
  • +
    + {item.id && ( + + #{item.id} + + )} + + {importanceLabels[item.importance]} + + + {evidenceLabels[item.evidenceType] || item.evidenceType} + + {item.confidence && } +
    +

    {item.description}

    + {(item.source || item.attribution) && ( +

    + Source: {item.source || item.attribution} +

    + )} +
    • + ))} +
+
+ ); +} + +// ── Main component ────────────────────────────────── export default function ReconstructionView({ reconstruction, partial }) { + // Handle both v0.2 direct object and wrapped result formats + const data = reconstruction; + if (partial) { return (
- ⚠ Partial result — some fields failed validation. Showing what was accepted. + ⚠ Partial result — some fields failed validation. Showing what was + accepted.
); } - const categories = Object.entries(categoryLabels).map(([key, label]) => ({ - key, - label, - items: reconstruction[key], - })); - return ( -
-

Reconstruction

- {categories.map(({ key, label, items }) => ( -
-

{label}

- -
- ))} +
+ {/* Classification first */} + {data.inputClassification && ( + + )} + + {/* Summary */} + {data.reconstruction?.summary && ( + + )} + + {/* Key differences */} + {data.reconstruction?.differences && ( + + )} + + {/* Unexplained transitions */} + {data.reconstruction?.unexplainedTransitions && + data.reconstruction.unexplainedTransitions.length > 0 && ( + + i.entity && ( +

Entity: {i.entity}

+ ) + } + /> + )} + + {/* Contradictions */} + {data.reconstruction?.contradictions && + data.reconstruction.contradictions.length > 0 && ( + + )} + + {/* Important unknowns */} + {data.reconstruction?.importantUnknowns && + data.reconstruction.importantUnknowns.length > 0 && ( + + )} + + {/* Plausible interpretations */} + {data.reconstruction?.plausibleInterpretations && + data.reconstruction.plausibleInterpretations.length > 0 && ( + + )} + + {/* Secondary reconstruction categories (actors, systems, etc.) */} + {data.reconstruction?.actors && data.reconstruction.actors.length > 0 && ( + + )} + {data.reconstruction?.systemsOrObjects && + data.reconstruction.systemsOrObjects.length > 0 && ( + + )} + {data.reconstruction?.expectedStates && + data.reconstruction.expectedStates.length > 0 && ( + + )} + {data.reconstruction?.observedStates && + data.reconstruction.observedStates.length > 0 && ( + + )} + {data.reconstruction?.knownTransitions && + data.reconstruction.knownTransitions.length > 0 && ( + ( +
+ {i.entity && Entity: {i.entity} · } + From “{i.previousState}” → To “{i.currentState}” ("{i.explanationStatus}") +
+ )} + /> + )} + + {/* Next question — prominent */} + + + {/* Evidence */} + {data.evidence && }
); } diff --git a/components/scenario-form.jsx b/components/scenario-form.jsx index 85bdd4c..fb2e7d8 100644 --- a/components/scenario-form.jsx +++ b/components/scenario-form.jsx @@ -8,7 +8,7 @@ const MAX_LENGTH = 10000; export default function ScenarioForm() { const [scenario, setScenario] = useState(""); - const [status, setStatus] = useState("idle"); // idle | loading | error | success + const [status, setStatus] = useState("idle"); // idle | loading | error | success | partial const [result, setResult] = useState(null); const textareaRef = useRef(null); @@ -29,6 +29,10 @@ export default function ScenarioForm() { if (res.ok && data.validationStatus === "valid") { setStatus("success"); setResult(data); + } else if (data.success) { + // Success in analysis but validation may be partial + setStatus("success"); + setResult(data); } else { setStatus("error"); setResult(data); @@ -39,8 +43,13 @@ export default function ScenarioForm() { } }; - // Always show diagnostics when there's a result (even if validation failed) - const hasDiagnostics = result && (result.reconstruction || result.modelName || result.responseDurationMs !== undefined); + // Determine if we have meaningful content to display + const hasClassification = result?.inputClassification; + const hasReconstruction = result?.reconstruction; + const hasNextQuestion = result?.nextQuestion; + const hasEvidence = result?.evidence && result.evidence.length > 0; + const hasMeaningfulContent = + hasClassification || hasReconstruction || hasNextQuestion || hasEvidence; return (
@@ -54,7 +63,9 @@ export default function ScenarioForm() { className="w-full rounded-lg border border-gray-300 px-4 py-3 text-sm focus:border-gray-500 focus:outline-none focus:ring-2 focus:ring-gray-400" />
- {scenario.length}/{MAX_LENGTH} + + {scenario.length}/{MAX_LENGTH} +
)} - {status === "success" && result?.reconstruction && ( + {/* Success state */} + {status === "success" && hasMeaningfulContent && (
- - +
)} - {status === "error" && result?.reconstruction && ( -
-
- ⚠ Partial result — some fields failed validation. Showing what was accepted. -
- -
+ {/* Always show diagnostics when we have any result */} + {(hasClassification || hasReconstruction || hasNextQuestion) && ( + )} {status === "loading" && ( -
Waiting for model response...
+
+ Waiting for model response... +
+ )} + + {/* Empty state */} + {status === "idle" && ( +
+

+ Enter a scenario above and click Analyse to begin. +

+
+ )} + + {/* Invalid result with no partial data */} + {status === "error" && !result?.error && !hasMeaningfulContent && ( +
+ Validation failed — no structured output was produced. +
)}
); diff --git a/lib/analysis.js b/lib/analysis.js new file mode 100644 index 0000000..8741cf1 --- /dev/null +++ b/lib/analysis.js @@ -0,0 +1,182 @@ +/** + * Core analysis pipeline — shared by API routes and evaluation harness. + * Calls the provider, parses output, validates against Zod schemas (v0.2 first, v0.1 fallback). + */ + +import { getConfig } from "../lib/config.js"; +import { getProvider } from "../lib/llm/provider.js"; +import { buildPrompt, PROMPT_VERSIONS } from "../lib/reconstruction/prompt.js"; +import { + reconstructionV2Schema, + reconstructionSchema as reconstructionV1Schema, +} from "../lib/reconstruction/schema.js"; + +const MAX_SCENARIO_LENGTH = 10000; +const DEFAULT_PROMPT_VERSION = "v0.2"; + +/** + * Analyse a scenario string through the full pipeline. + * @param {string} scenario - The scenario text to analyse + * @param {object} [opts] + * @param {"v0.1" | "v0.2"} [opts.promptVersion="v0.2"] - Prompt version to use + * @returns {Promise} Analysis result with diagnostics + */ +export async function analyseScenario(scenario, opts = {}) { + const startTime = Date.now(); + + // ── Input validation ─────────────────────────────── + if (typeof scenario !== "string") { + return buildErrorResponse("Input must be a string", startTime); + } + + const trimmed = scenario.trim(); + if (trimmed.length === 0) { + return buildErrorResponse("Scenario cannot be empty", startTime); + } + if (trimmed.length > MAX_SCENARIO_LENGTH) { + return buildErrorResponse(`Scenario must be under ${MAX_SCENARIO_LENGTH} characters`, startTime); + } + + // ── Configuration check ──────────────────────────── + const configResult = getConfig(); + if (!configResult.ok) { + return buildErrorResponse("Invalid server configuration", startTime, "500"); + } + + const { OLLAMA_BASE_URL: _ignored, OLLAMA_MODEL } = configResult.config; + const promptVersion = opts.promptVersion || DEFAULT_PROMPT_VERSION; + + // ── Build prompt ─────────────────────────────────── + let promptObj; + try { + promptObj = await buildPrompt(trimmed, promptVersion); + } catch (e) { + return buildErrorResponse(`Failed to build prompt: ${e.message}`, startTime); + } + + // ── Call provider ────────────────────────────────── + const provider = getProvider(); + let rawResponse; + try { + rawResponse = await provider.generateReconstruction(promptObj.prompt, OLLAMA_MODEL); + } catch (e) { + return buildErrorResponse( + e.message || "Provider error during analysis", + Date.now() - startTime + ); + } + + const duration = Date.now() - startTime; + + // Try to capture raw response for diagnostics + let rawResponseStr; + try { + rawResponseStr = JSON.stringify(rawResponse); + } catch { + rawResponseStr = String(rawResponse).slice(0, 2000); + } + + // ── Validate against v0.2 schema (preferred) ────── + const resultV2 = tryValidateAgainstSchema(rawResponse, reconstructionV2Schema); + if (resultV2.valid) { + return buildSuccessResultV2(resultV2.data, OLLAMA_MODEL, duration, promptVersion); + } + + // ── Fallback to v0.1 schema ──────────────────────── + const resultV1 = tryValidateAgainstSchema(rawResponse, reconstructionV1Schema); + if (resultV1.valid) { + return buildSuccessResultV1(resultV1.data, OLLAMA_MODEL, duration, promptVersion); + } + + // ── Neither schema matched — partial failure ─────── + return buildPartialResult( + rawResponseStr?.slice(0, 2000), + resultV2.error ?? resultV1.error, + OLLAMA_MODEL, + duration, + promptVersion + ); +} + +/** Attempt validation against a Zod schema */ +function tryValidateAgainstSchema(data, schema) { + if (!schema.safeParse) { + return { valid: false, error: new Error("Schema does not support safeParse") }; + } + const result = schema.safeParse(data); + return result.success ? { valid: true, data: result.data } : { valid: false, error: result.error }; +} + +// ── Result builders ────────────────────────────────── + +function buildErrorResponse(message, elapsed, statusCode = 500) { + return { + success: false, + error: message, + modelName: null, + responseDurationMs: elapsed, + validationStatus: "invalid", + rawResponse: null, + promptVersion: null, + statusCode, + }; +} + +function buildSuccessResultV2(data, model, duration, version) { + return { + success: true, + validationStatus: "valid", + modelName: model, + responseDurationMs: duration, + rawResponse: JSON.stringify(data).slice(0, 3000), + promptVersion: version, + inputClassification: data.inputClassification, + reconstruction: data.reconstruction, + evidence: data.evidence, + nextQuestion: data.nextQuestion, + errors: undefined, + }; +} + +function buildSuccessResultV1(data, model, duration, version) { + return { + success: true, + validationStatus: "valid", + modelName: model, + responseDurationMs: duration, + rawResponse: JSON.stringify(data).slice(0, 3000), + promptVersion: version, + inputClassification: null, + reconstruction: data, + evidence: undefined, + nextQuestion: undefined, + errors: undefined, + }; +} + +function buildPartialResult(rawResp, error, model, duration, version) { + let errors = []; + if (error && typeof error.flatten === "function") { + errors = error.flatten().fieldErrors + ? Object.entries(error.flatten().fieldErrors).flatMap(([k, v]) => [`${k}: ${v.join(", ")}`]) + : [String(error)]; + } else if (error) { + errors = [String(error).slice(0, 500)]; + } + + return { + success: false, + validationStatus: "invalid", + modelName: model, + responseDurationMs: duration, + rawResponse: rawResp?.slice(0, 2000), + promptVersion: version, + inputClassification: null, + reconstruction: null, + evidence: undefined, + nextQuestion: undefined, + errors, + }; +} + +export { PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION }; diff --git a/lib/llm/provider.js b/lib/llm/provider.js index 716a8cc..c41a313 100644 --- a/lib/llm/provider.js +++ b/lib/llm/provider.js @@ -93,11 +93,9 @@ async function detectChatSupport(baseUrl) { class OllamaLlmProvider { async generateReconstruction(scenario, modelName) { - const { buildPrompt } = await import("@/lib/reconstruction/prompt"); - - let rawPrompt = buildPrompt(scenario); - // Stronger JSON hint since we can't use format:json on older Ollama - const prompt = rawPrompt + `\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.`; + // scenario is ALREADY a fully-built prompt text (built by analyseScenario). + // Do NOT call buildPrompt() again — that would double-wrap the prompt. + const prompt = scenario; const baseUrl = process.env.OLLAMA_BASE_URL; if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set"); diff --git a/lib/reconstruction/prompt.js b/lib/reconstruction/prompt.js index 691700d..66edeab 100644 --- a/lib/reconstruction/prompt.js +++ b/lib/reconstruction/prompt.js @@ -1,5 +1,17 @@ -export function buildPrompt(scenario) { - return `You are a neutral analyst performing an evidence-based reconstruction of the following scenario. +import { promises as fs } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { dirname, join } from "node:path"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const PROMPTS_DIR = join(__dirname, "../../prompts"); + +/** Available prompt versions */ +export const PROMPT_VERSIONS = ["v0.1", "v0.2"]; + +/** Build a v0.1 (extraction-only) prompt inline for backward compatibility */ +function buildV1Prompt(scenario) { + return `You are a neutral analyst performing an evidence-based reconstruction of the following scenario. Rules: 1. Do NOT invent facts. Only include information present in the scenario or clearly implied. @@ -29,3 +41,38 @@ Return valid JSON matching this structure exactly: Return ONLY the JSON object. No markdown, no explanation, no preamble.`; } + +/** Load a versioned prompt from disk and substitute {{SCENARIO}} */ +async function buildV2Prompt(scenario) { + try { + const content = await fs.readFile( + join(PROMPTS_DIR, "reconstruct-v0.2.md"), + "utf-8", + ); + return content.replace("{{SCENARIO}}", scenario); + } catch { + // Fall back to v0.1 prompt if v0.2 file is missing + return buildV1Prompt(scenario); + } +} + +/** + * Build an analysis prompt for the given version. + * @param {"v0.1" | "v0.2"} [version="v0.2"] + * @returns {Promise<{prompt: string, version: string}>} + */ +export async function buildPrompt(scenario, version = "v0.2") { + let prompt; + switch (version) { + case "v0.1": + prompt = buildV1Prompt(scenario); + break; + default: // v0.2 + prompt = await buildV2Prompt(scenario); + break; + } + + const strongJsonHint = + "\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks."; + return { prompt: prompt + strongJsonHint, version }; +} diff --git a/lib/reconstruction/schema.js b/lib/reconstruction/schema.js index f94e831..d6f237f 100644 --- a/lib/reconstruction/schema.js +++ b/lib/reconstruction/schema.js @@ -1,38 +1,59 @@ import { z } from "zod"; -const confidenceEnum = z.enum(["low", "medium", "high"]); +// ────────────────────────────────────────────── +// Shared enums (v0.1 & v0.2) +// ────────────────────────────────────────────── -const itemSchema = z.object({ +export const confidenceEnum = z.enum(["low", "medium", "high"]); +const importanceEnum = z.enum([ + "incidental", + "supporting", + "important", + "critical", +]); +const expectedInfoValueEnum = z.enum(["low", "medium", "high"]); + +// ────────────────────────────────────────────── +// v0.1 — extraction-only schema (preserved) +// ────────────────────────────────────────────── + +const confidenceEnumV1 = z.enum(["low", "medium", "high"]); + +const itemSchemaV1 = z.object({ id: z.string().min(1), description: z.string().min(1), - confidence: confidenceEnum, + confidence: confidenceEnumV1, }); export const reconstructionSchema = z.object({ - observations: z.array(itemSchema), + observations: z.array(itemSchemaV1), reportedClaims: z.array( - itemSchema.extend({ - attributedTo: z.union([z.string().min(1), z.null()]).optional().nullable(), - }) + itemSchemaV1.extend({ + attributedTo: z + .union([z.string().min(1), z.null()]) + .optional() + .nullable(), + }), ), - assumptions: z.array(itemSchema), - entities: z.array(itemSchema), + assumptions: z.array(itemSchemaV1), + entities: z.array(itemSchemaV1), transitions: z.array( - itemSchema.extend({ + itemSchemaV1.extend({ entity: z.string().min(1), previousState: z.string().min(1), currentState: z.string().min(1), explanationStatus: z.string().min(1), - }) + }), ), - expectedButMissing: z.array(itemSchema), - presentButUnexpected: z.array(itemSchema), - contradictions: z.array(itemSchema), - openUncertainties: z.array(itemSchema), + expectedButMissing: z.array(itemSchemaV1), + presentButUnexpected: z.array(itemSchemaV1), + contradictions: z.array(itemSchemaV1), + openUncertainties: z.array(itemSchemaV1), }); +// v0.1 analyse response (used internally) export const analyseResponseSchema = z.object({ - reconstruction: reconstructionSchema, + reconstruction: z.union([reconstructionSchema, z.null()]), modelName: z.string(), responseDurationMs: z.number(), validationStatus: z.enum(["valid", "partial", "invalid"]), @@ -48,6 +69,141 @@ export const healthResponseSchema = z.object({ error: z.string().nullable(), }); +// ────────────────────────────────────────────── +// v0.2 — reasoning classification + reconstruction +// ────────────────────────────────────────────── + +export const inputTypes = + /** @type {z.ZodType} */ ( + z.enum([ + "observed_problem", + "unexplained_change", + "contradiction", + "decision_request", + "causal_claim", + "reported_claim", + "fault_report", + "ambiguous_statement", + "question", + "desired_outcome", + "insufficient_context", + "other", + ]) + ); + +export const reasoningModes = + /** @type {z.ZodType} */ ( + z.enum([ + "establish_baseline", + "identify_difference", + "reconstruct_transition", + "decompose_aggregate", + "validate_measurement", + "validate_claim", + "investigate_contradiction", + "clarify_meaning", + "decision_support", + "fault_investigation", + "identify_missing_information", + "test_possible_explanations", + "other", + ]) + ); + +const evidenceRecordSchema = z.object({ + id: z.string().min(1), + description: z.string().min(1), + evidenceType: z.enum([ + "direct_observation", + "reported_statement", + "interpretation", + "assumption", + "inferred_relationship", + ]), + source: z.string().optional(), + attribution: z.string().nullable().optional(), + confidence: confidenceEnum, + importance: importanceEnum, +}); + +const reconstructionSchemaV2 = z.object({ + summary: z.string().min(1), + actors: z.array(itemSchemaV1), + systemsOrObjects: z.array(itemSchemaV1), + expectedStates: z.array(itemSchemaV1), + observedStates: z.array(itemSchemaV1), + differences: z.array(itemSchemaV1), + knownTransitions: z.array( + itemSchemaV1.extend({ + entity: z.string().min(1), + previousState: z.string().min(1), + currentState: z.string().min(1), + explanationStatus: z.string().min(1), + }), + ), + unexplainedTransitions: z.array( + itemSchemaV1.extend({ + entity: z.string().min(1).optional(), + previousState: z.string().min(1).optional(), + currentState: z.string().min(1).optional(), + }), + ), + contradictions: z.array(itemSchemaV1), + importantUnknowns: z.array(itemSchemaV1), + plausibleInterpretations: z.array( + z.object({ + id: z.string().min(1), + description: z.string().min(1), + supportingEvidenceIds: z.array(z.string()), + assumptionsRequired: z.array(z.string()).optional().default([]), + confidence: confidenceEnum, + }), + ), +}); + +const inputClassificationSchema = z.object({ + primaryType: inputTypes, + secondaryTypes: z.array(inputTypes).optional().default([]), + reasoningModes: z.array(reasoningModes).optional().default([]), + classificationReason: z.string().min(1), + confidence: confidenceEnum, +}); + +const nextQuestionSchema = z.object({ + id: z.string().min(1), + question: z.string().min(1), + targets: z.array(z.string()), + reason: z.string().min(1), + expectedInformationValue: expectedInfoValueEnum, + reasoningMode: reasoningModes.optional().default("other"), +}); + +// v0.2 complete analysis response (what the model produces) +export const reconstructionV2Schema = z.object({ + inputClassification: inputClassificationSchema, + reconstruction: reconstructionSchemaV2, + evidence: z.array(evidenceRecordSchema), + nextQuestion: nextQuestionSchema, +}); + +// Outer wrapper for API return (includes diagnostics + v0.2 data) +export const analyseResponseV2Schema = z.object({ + inputClassification: inputClassificationSchema.optional(), + reconstruction: reconstructionSchemaV2.optional().nullable(), + evidence: z.array(evidenceRecordSchema).optional(), + nextQuestion: nextQuestionSchema.optional(), + modelName: z.string(), + responseDurationMs: z.number(), + validationStatus: z.enum(["valid", "partial", "invalid"]), + rawResponse: z.string().optional(), + errors: z.array(z.string()).optional(), + promptVersion: z.string().optional(), +}); + +// ────────────────────────────────────────────── +// Parsing helpers +// ────────────────────────────────────────────── + export function parseReconstruction(raw) { if (typeof raw === "string") { try { @@ -58,3 +214,14 @@ export function parseReconstruction(raw) { } return reconstructionSchema.parse(raw); } + +export function parseReconstructionV2(raw) { + if (typeof raw === "string") { + try { + raw = JSON.parse(raw); + } catch { + throw new SyntaxError("Model response is not valid JSON"); + } + } + return reconstructionV2Schema.parse(raw); +} diff --git a/package.json b/package.json index e3827f3..4f1c4e6 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "type": "module", "name": "confidence-engine", - "version": "0.1.0", + "version": "0.2.0-experimental", "private": true, "description": "Experimental prototype for evidence-based situation reconstruction using local LLMs", "scripts": { diff --git a/prompts/reconstruct-v0.2.md b/prompts/reconstruct-v0.2.md new file mode 100644 index 0000000..6d0f7dd --- /dev/null +++ b/prompts/reconstruct-v0.2.md @@ -0,0 +1,122 @@ +You are a neutral analyst performing evidence-based situation reconstruction. + +## Rules + +1. Do NOT invent facts, context or causes. Only include information present in the scenario or clearly implied. +2. First determine what kind of input has been supplied. Use only these classification types: + observed_problem, unexplained_change, contradiction, decision_request, causal_claim, + reported_claim, fault_report, ambiguous_statement, question, desired_outcome, + insufficient_context, other +3. Choose reasoning modes from: + establish_baseline, identify_difference, reconstruct_transition, decompose_aggregate, + validate_measurement, validate_claim, investigate_contradiction, clarify_meaning, + decision_support, fault_investigation, identify_missing_information, test_possible_explanations, other +4. Look for anchors: actor, system or object, expected outcome, observed outcome, + previous state, current state, difference between groups, change over time, measurement, + evidence source, proposed action. +5. Identify meaningful differences (e.g., some succeed while others fail; revenue rises while cash falls). +6. Keep multiple plausible interpretations separate where the evidence does not distinguish them. +7. Distinguish: what was said / what it may mean / why it may have been said. +8. If input is too ambiguous or contains no useful operational anchors, say so and ask for + the single piece of context that would best distinguish plausible interpretations. + +## Confidence scale + +- low — weak evidence, speculation, or missing information +- medium — reasonable inference from available evidence +- high — strong evidence, direct observation, or confirmed fact + +## Importance scale (evidence records) + +- incidental — minor detail, unlikely to affect conclusions +- supporting — adds context but not critical +- important — materially affects understanding of the situation +- critical — essential to resolving the situation; without it conclusions cannot be drawn + +## Expected information value (next question) + +- low — marginally useful even if answered +- medium — meaningfully clarifies the situation +- high — would significantly distinguish between plausible explanations or fill a gap in understanding + +## Next question selection criteria + +Prefer questions that: +- clarify a major difference +- establish a baseline +- explain an important transition +- test an unsupported claim +- distinguish between plausible explanations +- request measurable evidence +- identify who or what is affected +- establish timing + +Avoid questions that: +- have already been answered +- assume a cause +- jump to a solution +- ask about motive before the observable situation is understood +- focus on incidental wording +- are too broad to produce useful information +- combine many unrelated questions + +## Output format — return this exact JSON structure + +Return a JSON object with exactly these four top-level keys (use **camelCase**): + +```json +{ + "inputClassification": { + "primaryType": "", + "secondaryTypes": [""], + "reasoningModes": [""], + "classificationReason": "", + "confidence": "" + }, + "reconstruction": { + "summary": "", + "actors": [{"id": "", "description": "...", "confidence": ""}], + "systemsOrObjects": [{"id": "", "description": "...", "confidence": ""}], + "expectedStates": [{"id": "...", "description": "...", "confidence": ""}], + "observedStates": [{"id": "...", "description": "...", "confidence": ""}], + "differences": [{"id": "...", "description": "...", "confidence": ""}], + "knownTransitions": [{"id": "...", "description": "...", "confidence": "", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}], + "unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "", "entity": "...", "previousState": "...", "currentState": "..."}], + "contradictions": [{"id": "...", "description": "...", "confidence": ""}], + "importantUnknowns": [{"id": "...", "description": "...", "confidence": ""}], + "plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": [""], "assumptionsRequired": [], "confidence": ""}] + }, + "evidence": [ + { + "id": "", + "description": "...", + "evidenceType": "", + "source": "", + "attribution": null, + "confidence": "", + "importance": "" + } + ], + "nextQuestion": { + "id": "", + "question": "", + "targets": [""], + "reason": "", + "expectedInformationValue": "", + "reasoningMode": "" + } +} +``` + +CRITICAL RULES for JSON output: +1. Use **exactly** the key names shown above (camelCase, no snake_case). +2. The four top-level keys must be: `inputClassification`, `reconstruction`, `evidence`, `nextQuestion`. +3. Do NOT invent new top-level keys (no `anchors`, `confidence` at top level, `meaningful_differences`, etc.). +4. Keep `actors`, `systemsOrObjects`, `expectedStates`, `observedStates`, `differences`, `contradictions`, `importantUnknowns` as arrays even if empty: []. +5. Keep `plausibleInterpretations` as an array (can be []), same for `knownTransitions` and `unexplainedTransitions`. +6. Each object in arrays must have at least `id`, `description`, `confidence`. + +Scenario: +{{SCENARIO}} + +Return ONLY the JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.