confidence-engine/lib/analysis.js

/**
 * Core analysis pipeline — shared by API routes and evaluation harness.
 * Calls the provider, parses output, validates against Zod schemas (v0.2 first, v0.1 fallback).
 */

import { getConfig } from "../lib/config.js";
import { getProvider, getProviderModelName } from "../lib/llm/provider.js";
import {
  buildPrompt,
  PROMPT_VERSIONS,
  DEFAULT_PROMPT_VERSION,
} from "../lib/reconstruction/prompt.js";
import { normaliseAnalysisResponse } from "../lib/reconstruction/compatibility.js";
import {
  reconstructionV2Schema,
  reconstructionSchema as reconstructionV1Schema,
} from "../lib/reconstruction/schema.js";

const MAX_SCENARIO_LENGTH = 10000;

/**
 * Analyse a scenario string through the full pipeline.
 * @param {string} scenario - The scenario text to analyse
 * @param {object} [opts]
 * @param {"v0.1" | "v0.2"} [opts.promptVersion="v0.2"] - Prompt version to use
 * @param {{ generateReconstruction: Function }} [opts.reconstructionProvider] - Experiment-only reconstruction provider override
 * @param {string} [opts.reconstructionModelName] - Experiment-only model override for an injected provider
 * @returns {Promise<object>} Analysis result with diagnostics
 */
export async function analyseScenario(scenario, opts = {}) {
  const startTime = Date.now();

  // ── Input validation ───────────────────────────────
  if (typeof scenario !== "string") {
    return buildErrorResponse("Input must be a string", startTime);
  }

  const trimmed = scenario.trim();
  if (trimmed.length === 0) {
    return buildErrorResponse("Scenario cannot be empty", startTime);
  }
  if (trimmed.length > MAX_SCENARIO_LENGTH) {
    return buildErrorResponse(
      `Scenario must be under ${MAX_SCENARIO_LENGTH} characters`,
      startTime,
    );
  }

  // ── Configuration check ────────────────────────────
  const configResult = getConfig();
  if (!configResult.ok) {
    return buildErrorResponse("Invalid server configuration", startTime, "500");
  }

  const promptVersion = opts.promptVersion || DEFAULT_PROMPT_VERSION;

  // ── Build prompt (experiment seam via env var bridge) ──
  let promptObj;
  try {
    const experimentInstruction = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION;
    const buildOpts = {};
    if (experimentInstruction) {
      buildOpts.experimentInstruction = experimentInstruction;
    }
    promptObj = await buildPrompt(trimmed, promptVersion, buildOpts);
  } catch (e) {
    return buildErrorResponse(
      `Failed to build prompt: ${e.message}`,
      startTime,
    );
  }

  // ── Call provider ──────────────────────────────────
  const provider = opts.reconstructionProvider ?? getProvider();
  const reconstructionModelName = opts.reconstructionModelName ?? getProviderModelName();
  let rawResponse;
  let providerApiPath;
  let providerExecution;
  try {
    const providerResult = await provider.generateReconstruction(
      promptObj.prompt,
      reconstructionModelName,
    );
    if (
      providerResult &&
      typeof providerResult === "object" &&
      "response" in providerResult &&
      "providerApiPath" in providerResult
    ) {
      rawResponse = providerResult.response;
      providerApiPath = providerResult.providerApiPath;
      providerExecution = providerResult.providerExecution;
    } else {
      rawResponse = providerResult;
    }
  } catch (e) {
    return buildErrorResponse(
      e.message || "Provider error during analysis",
      Date.now() - startTime,
      "500",
      e.providerApiPath,
      e.providerExecution,
      e.code,
    );
  }

  const duration = Date.now() - startTime;

  // Try to capture raw response for diagnostics
  let rawResponseStr;
  try {
    rawResponseStr = JSON.stringify(rawResponse);
  } catch {
    rawResponseStr = String(rawResponse);
  }

  const compatibility = normaliseAnalysisResponse(rawResponse);
  const candidateResponse = compatibility.normalised;

  // ── Validate against v0.2 schema (preferred) ──────
  const resultV2 = tryValidateAgainstSchema(
    candidateResponse,
    reconstructionV2Schema,
  );
  if (resultV2.valid) {
    return buildSuccessResultV2(
      resultV2.data,
      reconstructionModelName,
      duration,
      promptVersion,
      compatibility,
    );
  }

  // ── Fallback to v0.1 schema ────────────────────────
  const resultV1 = tryValidateAgainstSchema(
    candidateResponse,
    reconstructionV1Schema,
  );
  if (resultV1.valid) {
    return buildSuccessResultV1(
      resultV1.data,
      reconstructionModelName,
      duration,
      promptVersion,
      compatibility,
    );
  }

  // ── Neither schema matched — partial failure ───────
  return buildPartialResult(
    rawResponseStr,
    resultV2.error ?? resultV1.error,
    reconstructionModelName,
    duration,
    promptVersion,
    compatibility,
    providerApiPath,
    providerExecution,
  );
}

/** Attempt validation against a Zod schema */
function tryValidateAgainstSchema(data, schema) {
  if (!schema.safeParse) {
    return {
      valid: false,
      error: new Error("Schema does not support safeParse"),
    };
  }
  const result = schema.safeParse(data);
  return result.success
    ? { valid: true, data: result.data }
    : { valid: false, error: result.error };
}

// ── Result builders ──────────────────────────────────

function buildErrorResponse(message, elapsed, statusCode = 500, providerApiPath, providerExecution, code) {
  return {
    success: false,
    error: message,
    modelName: null,
    responseDurationMs: elapsed,
    validationStatus: "invalid",
    rawResponse: null,
    promptVersion: null,
    statusCode,
    providerApiPath,
    providerExecution,
    code,
  };
}

function buildCompatibilityDiagnostics(compatibility) {
  return {
    compatibilityApplied: compatibility.changesApplied.length > 0,
    compatibilityChanges: compatibility.changesApplied,
    compatibilityWarnings: compatibility.warnings,
  };
}

function buildSuccessResultV2(data, model, duration, version, compatibility) {
  return {
    success: true,
    validationStatus: "valid",
    modelName: model,
    responseDurationMs: duration,
    rawResponse: JSON.stringify(data).slice(0, 3000),
    promptVersion: version,
    inputClassification: data.inputClassification,
    reconstruction: data.reconstruction,
    evidence: data.evidence,
    nextQuestion: data.nextQuestion,
    errors: undefined,
    ...buildCompatibilityDiagnostics(compatibility),
  };
}

function buildSuccessResultV1(data, model, duration, version, compatibility) {
  return {
    success: true,
    validationStatus: "valid",
    modelName: model,
    responseDurationMs: duration,
    rawResponse: JSON.stringify(data).slice(0, 3000),
    promptVersion: version,
    inputClassification: null,
    reconstruction: data,
    evidence: undefined,
    nextQuestion: undefined,
    errors: undefined,
    ...buildCompatibilityDiagnostics(compatibility),
  };
}

function buildPartialResult(
  rawResp,
  error,
  model,
  duration,
  version,
  compatibility,
  providerApiPath,
  providerExecution,
) {
  let errors = [];
  const validationIssues = error?.issues ?? [];
  if (error && typeof error.flatten === "function") {
    errors = error.flatten().fieldErrors
      ? Object.entries(error.flatten().fieldErrors).flatMap(([k, v]) => [
          `${k}: ${v.join(", ")}`,
        ])
      : [String(error)];
  } else if (error) {
    errors = [String(error).slice(0, 500)];
  }

  return {
    success: false,
    validationStatus: "invalid",
    modelName: model,
    responseDurationMs: duration,
    rawResponse: rawResp,
    promptVersion: version,
    inputClassification: null,
    reconstruction: null,
    evidence: undefined,
    nextQuestion: undefined,
    errors,
    validationIssues,
    providerApiPath,
    providerExecution,
    ...buildCompatibilityDiagnostics(compatibility),
  };
}

export { PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION };