confidence-engine/lib/analysis.js

/**
 * Core analysis pipeline — shared by API routes and evaluation harness.
 * Calls the provider, parses output, validates against Zod schemas (v0.2 first, v0.1 fallback).
 */

import { getConfig } from "../lib/config.js";
import { getProvider } from "../lib/llm/provider.js";
import { buildPrompt, PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION } from "../lib/reconstruction/prompt.js";
import {
  reconstructionV2Schema,
  reconstructionSchema as reconstructionV1Schema,
} from "../lib/reconstruction/schema.js";

const MAX_SCENARIO_LENGTH = 10000;

/**
 * Analyse a scenario string through the full pipeline.
 * @param {string} scenario - The scenario text to analyse
 * @param {object} [opts]
 * @param {"v0.1" | "v0.2"} [opts.promptVersion="v0.2"] - Prompt version to use
 * @returns {Promise<object>} Analysis result with diagnostics
 */
export async function analyseScenario(scenario, opts = {}) {
  const startTime = Date.now();

  // ── Input validation ───────────────────────────────
  if (typeof scenario !== "string") {
    return buildErrorResponse("Input must be a string", startTime);
  }

  const trimmed = scenario.trim();
  if (trimmed.length === 0) {
    return buildErrorResponse("Scenario cannot be empty", startTime);
  }
  if (trimmed.length > MAX_SCENARIO_LENGTH) {
    return buildErrorResponse(`Scenario must be under ${MAX_SCENARIO_LENGTH} characters`, startTime);
  }

  // ── Configuration check ────────────────────────────
  const configResult = getConfig();
  if (!configResult.ok) {
    return buildErrorResponse("Invalid server configuration", startTime, "500");
  }

  const { OLLAMA_BASE_URL: _ignored, OLLAMA_MODEL } = configResult.config;
  const promptVersion = opts.promptVersion || DEFAULT_PROMPT_VERSION;

  // ── Build prompt ───────────────────────────────────
  let promptObj;
  try {
    promptObj = await buildPrompt(trimmed, promptVersion);
  } catch (e) {
    return buildErrorResponse(`Failed to build prompt: ${e.message}`, startTime);
  }

  // ── Call provider ──────────────────────────────────
  const provider = getProvider();
  let rawResponse;
  try {
    rawResponse = await provider.generateReconstruction(promptObj.prompt, OLLAMA_MODEL);
  } catch (e) {
    return buildErrorResponse(
      e.message || "Provider error during analysis",
      Date.now() - startTime
    );
  }

  const duration = Date.now() - startTime;

  // Try to capture raw response for diagnostics
  let rawResponseStr;
  try {
    rawResponseStr = JSON.stringify(rawResponse);
  } catch {
    rawResponseStr = String(rawResponse).slice(0, 2000);
  }

  // ── Validate against v0.2 schema (preferred) ──────
  const resultV2 = tryValidateAgainstSchema(rawResponse, reconstructionV2Schema);
  if (resultV2.valid) {
    return buildSuccessResultV2(resultV2.data, OLLAMA_MODEL, duration, promptVersion);
  }

  // ── Fallback to v0.1 schema ────────────────────────
  const resultV1 = tryValidateAgainstSchema(rawResponse, reconstructionV1Schema);
  if (resultV1.valid) {
    return buildSuccessResultV1(resultV1.data, OLLAMA_MODEL, duration, promptVersion);
  }

  // ── Neither schema matched — partial failure ───────
  return buildPartialResult(
    rawResponseStr?.slice(0, 2000),
    resultV2.error ?? resultV1.error,
    OLLAMA_MODEL,
    duration,
    promptVersion
  );
}

/** Attempt validation against a Zod schema */
function tryValidateAgainstSchema(data, schema) {
  if (!schema.safeParse) {
    return { valid: false, error: new Error("Schema does not support safeParse") };
  }
  const result = schema.safeParse(data);
  return result.success ? { valid: true, data: result.data } : { valid: false, error: result.error };
}

// ── Result builders ──────────────────────────────────

function buildErrorResponse(message, elapsed, statusCode = 500) {
  return {
    success: false,
    error: message,
    modelName: null,
    responseDurationMs: elapsed,
    validationStatus: "invalid",
    rawResponse: null,
    promptVersion: null,
    statusCode,
  };
}

function buildSuccessResultV2(data, model, duration, version) {
  return {
    success: true,
    validationStatus: "valid",
    modelName: model,
    responseDurationMs: duration,
    rawResponse: JSON.stringify(data).slice(0, 3000),
    promptVersion: version,
    inputClassification: data.inputClassification,
    reconstruction: data.reconstruction,
    evidence: data.evidence,
    nextQuestion: data.nextQuestion,
    errors: undefined,
  };
}

function buildSuccessResultV1(data, model, duration, version) {
  return {
    success: true,
    validationStatus: "valid",
    modelName: model,
    responseDurationMs: duration,
    rawResponse: JSON.stringify(data).slice(0, 3000),
    promptVersion: version,
    inputClassification: null,
    reconstruction: data,
    evidence: undefined,
    nextQuestion: undefined,
    errors: undefined,
  };
}

function buildPartialResult(rawResp, error, model, duration, version) {
  let errors = [];
  if (error && typeof error.flatten === "function") {
    errors = error.flatten().fieldErrors
      ? Object.entries(error.flatten().fieldErrors).flatMap(([k, v]) => [`${k}: ${v.join(", ")}`])
      : [String(error)];
  } else if (error) {
    errors = [String(error).slice(0, 500)];
  }

  return {
    success: false,
    validationStatus: "invalid",
    modelName: model,
    responseDurationMs: duration,
    rawResponse: rawResp?.slice(0, 2000),
    promptVersion: version,
    inputClassification: null,
    reconstruction: null,
    evidence: undefined,
    nextQuestion: undefined,
    errors,
  };
}

export { PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION };