chore: establish clean v0.2 baseline

Include only the working reconstruction prototype with Ollama integration: - double-wrapping fix (lib/llm/provider.js) - explicit v0.2 JSON output schema (prompts/reconstruct-v0.2.md) - Zod validation layer (lib/reconstruction/schema.js) - shared core analysis path (lib/analysis.js) - prompt versioning infrastructure (lib/reconstruction/prompt.js) - provider abstraction - functioning Ollama provider path - updated API route with centralized analysis - UI components displaying v0.2 data and validation errors - .gitignore rules for generated evaluation artifacts Exclude: evaluator experiments, diagnostic tests, debug scripts, generated artifacts, comparison findings, test data tied to evaluator.
2026-08-01 14:45:06 +01:00
parent a2f9e472ea
commit d72c7c5465
11 changed files with 1048 additions and 172 deletions
@@ -0,0 +1,182 @@
+/**
+ * Core analysis pipeline — shared by API routes and evaluation harness.
+ * Calls the provider, parses output, validates against Zod schemas (v0.2 first, v0.1 fallback).
+ */
+
+import { getConfig } from "../lib/config.js";
+import { getProvider } from "../lib/llm/provider.js";
+import { buildPrompt, PROMPT_VERSIONS } from "../lib/reconstruction/prompt.js";
+import {
+  reconstructionV2Schema,
+  reconstructionSchema as reconstructionV1Schema,
+} from "../lib/reconstruction/schema.js";
+
+const MAX_SCENARIO_LENGTH = 10000;
+const DEFAULT_PROMPT_VERSION = "v0.2";
+
+/**
+ * Analyse a scenario string through the full pipeline.
+ * @param {string} scenario - The scenario text to analyse
+ * @param {object} [opts]
+ * @param {"v0.1" | "v0.2"} [opts.promptVersion="v0.2"] - Prompt version to use
+ * @returns {Promise<object>} Analysis result with diagnostics
+ */
+export async function analyseScenario(scenario, opts = {}) {
+  const startTime = Date.now();
+
+  // ── Input validation ───────────────────────────────
+  if (typeof scenario !== "string") {
+    return buildErrorResponse("Input must be a string", startTime);
+  }
+
+  const trimmed = scenario.trim();
+  if (trimmed.length === 0) {
+    return buildErrorResponse("Scenario cannot be empty", startTime);
+  }
+  if (trimmed.length > MAX_SCENARIO_LENGTH) {
+    return buildErrorResponse(`Scenario must be under ${MAX_SCENARIO_LENGTH} characters`, startTime);
+  }
+
+  // ── Configuration check ────────────────────────────
+  const configResult = getConfig();
+  if (!configResult.ok) {
+    return buildErrorResponse("Invalid server configuration", startTime, "500");
+  }
+
+  const { OLLAMA_BASE_URL: _ignored, OLLAMA_MODEL } = configResult.config;
+  const promptVersion = opts.promptVersion || DEFAULT_PROMPT_VERSION;
+
+  // ── Build prompt ───────────────────────────────────
+  let promptObj;
+  try {
+    promptObj = await buildPrompt(trimmed, promptVersion);
+  } catch (e) {
+    return buildErrorResponse(`Failed to build prompt: ${e.message}`, startTime);
+  }
+
+  // ── Call provider ──────────────────────────────────
+  const provider = getProvider();
+  let rawResponse;
+  try {
+    rawResponse = await provider.generateReconstruction(promptObj.prompt, OLLAMA_MODEL);
+  } catch (e) {
+    return buildErrorResponse(
+      e.message || "Provider error during analysis",
+      Date.now() - startTime
+    );
+  }
+
+  const duration = Date.now() - startTime;
+
+  // Try to capture raw response for diagnostics
+  let rawResponseStr;
+  try {
+    rawResponseStr = JSON.stringify(rawResponse);
+  } catch {
+    rawResponseStr = String(rawResponse).slice(0, 2000);
+  }
+
+  // ── Validate against v0.2 schema (preferred) ──────
+  const resultV2 = tryValidateAgainstSchema(rawResponse, reconstructionV2Schema);
+  if (resultV2.valid) {
+    return buildSuccessResultV2(resultV2.data, OLLAMA_MODEL, duration, promptVersion);
+  }
+
+  // ── Fallback to v0.1 schema ────────────────────────
+  const resultV1 = tryValidateAgainstSchema(rawResponse, reconstructionV1Schema);
+  if (resultV1.valid) {
+    return buildSuccessResultV1(resultV1.data, OLLAMA_MODEL, duration, promptVersion);
+  }
+
+  // ── Neither schema matched — partial failure ───────
+  return buildPartialResult(
+    rawResponseStr?.slice(0, 2000),
+    resultV2.error ?? resultV1.error,
+    OLLAMA_MODEL,
+    duration,
+    promptVersion
+  );
+}
+
+/** Attempt validation against a Zod schema */
+function tryValidateAgainstSchema(data, schema) {
+  if (!schema.safeParse) {
+    return { valid: false, error: new Error("Schema does not support safeParse") };
+  }
+  const result = schema.safeParse(data);
+  return result.success ? { valid: true, data: result.data } : { valid: false, error: result.error };
+}
+
+// ── Result builders ──────────────────────────────────
+
+function buildErrorResponse(message, elapsed, statusCode = 500) {
+  return {
+    success: false,
+    error: message,
+    modelName: null,
+    responseDurationMs: elapsed,
+    validationStatus: "invalid",
+    rawResponse: null,
+    promptVersion: null,
+    statusCode,
+  };
+}
+
+function buildSuccessResultV2(data, model, duration, version) {
+  return {
+    success: true,
+    validationStatus: "valid",
+    modelName: model,
+    responseDurationMs: duration,
+    rawResponse: JSON.stringify(data).slice(0, 3000),
+    promptVersion: version,
+    inputClassification: data.inputClassification,
+    reconstruction: data.reconstruction,
+    evidence: data.evidence,
+    nextQuestion: data.nextQuestion,
+    errors: undefined,
+  };
+}
+
+function buildSuccessResultV1(data, model, duration, version) {
+  return {
+    success: true,
+    validationStatus: "valid",
+    modelName: model,
+    responseDurationMs: duration,
+    rawResponse: JSON.stringify(data).slice(0, 3000),
+    promptVersion: version,
+    inputClassification: null,
+    reconstruction: data,
+    evidence: undefined,
+    nextQuestion: undefined,
+    errors: undefined,
+  };
+}
+
+function buildPartialResult(rawResp, error, model, duration, version) {
+  let errors = [];
+  if (error && typeof error.flatten === "function") {
+    errors = error.flatten().fieldErrors
+      ? Object.entries(error.flatten().fieldErrors).flatMap(([k, v]) => [`${k}: ${v.join(", ")}`])
+      : [String(error)];
+  } else if (error) {
+    errors = [String(error).slice(0, 500)];
+  }
+
+  return {
+    success: false,
+    validationStatus: "invalid",
+    modelName: model,
+    responseDurationMs: duration,
+    rawResponse: rawResp?.slice(0, 2000),
+    promptVersion: version,
+    inputClassification: null,
+    reconstruction: null,
+    evidence: undefined,
+    nextQuestion: undefined,
+    errors,
+  };
+}
+
+export { PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION };