confidence-engine/tests/graph/live-update-experiment-helper.cjs

/**
 * Canonical live-update experiment harness.
 *
 * Loads .env.local, validates Ollama configuration, invokes updateCase()
 * through the real production path, and captures the standard reasoning
 * checkpoints needed by reasoning experiments.
 *
 * Usage (from a test file):
 *   const { runLiveExperiment } = require("./tests/graph/live-update-experiment-helper.cjs");
 *
 *   const result = await runLiveExperiment({
 *     graph: /* SituationGraph fixture *\/,
 *     previousQuestion: "Is risk a hard constraint?",
 *     answer: "Risk matters more to me.",
 *   });
 */

const dotenv = require("dotenv");
dotenv.config({ path: ".env.local" });

function assertRequiredEnv(name) {
  const value = process.env[name];
  if (!value) {
    throw new Error(
      `Live experiment requires ${name}. Set it in .env.local.\n` +
      `Found: OLLAMA_BASE_URL=${process.env.OLLAMA_BASE_URL ?? "(missing)"}, ` +
      `OLLAMA_MODEL=${process.env.OLLAMA_MODEL ?? "(missing)"}`
    );
  }
  return value;
}

/**
 * Extract checkpoint data from a raw updateCase() result object.
 *
 * Pure function — no side effects, no I/O. Intended for use both by
 * `runLiveExperiment()` (apparatus wrapper) and by deterministic tests
 * that pass mock results directly.
 *
 * @param {object} result  — raw result from updateCase()
 * @param {object} params — { graph?, answer? }
 *   - graph: SituationGraph input fixture (for findingsCount total derived from input)
 *   - answer: previous question text for deriveCategory fallback
 * @returns {object} checkpoint object per spec contract
 */
function extractLiveExperimentCheckpoint(result, params) {
  const answerMeaning = result?.proposal?.answerMeaning || {};

  // Graph structural checkpoint (available from updatedSituationGraph only when present)
  const updatedGraph = result?.updatedSituationGraph;
  const graphCheckpoint = updatedGraph
    ? {
        nodeCount:       updatedGraph.nodes?.length      ?? 0,
        edgeCount:       updatedGraph.edges?.length      ?? 0,
        resolvedNodeIds: updatedGraph.resolvedNodeIds    ?? [],
      }
    : null;

  return {
    // ── timing (available via wrapper) ────────────────────
    modelElapsedMs:       result?.modelElapsedMs       !== undefined ? result.modelElapsedMs : null,
    endToEndElapsedMs:    result?.endToEndElapsedMs    !== undefined ? result.endToEndElapsedMs : null,
    providerGenerateCalls: result != null && result.providerGenerateCalls != null ? result.providerGenerateCalls : 0,
    modelName:            result?.modelName             || null,

    // ── answer meaning extraction ─────────────────────────
    userSupportedMeaning: answerMeaning.userSupportedMeaning ?? null,
    possibleInference:    answerMeaning.possibleInference    ?? null,
    rawAnswerCategory:    _deriveCategory(params?.answer),
    proposedMeaningCategory: _deriveCategory(answerMeaning.userSupportedMeaning || ""),

    // ── validation / guard checkpoints ───────────────────
    proposalValidation: result?.proposalValidation ?? { success: false, errors: [] },
    compatibilityGuard: result?.compatibilityApplied !== undefined
      ? { passed: result.compatibilityApplied, warnings: [], violations: [] }
      : { passed: false, warnings: [], violations: [] },

    // ── graph mutation checkpoint (from appliedGraph) ────
    graphMutation: result?.appliedGraph
      ? { nodes: result.appliedGraph.nodes ?? null, edges: result.appliedGraph.edges ?? null }
      : null,

    // ── graph structural checkpoint (available from updatedSituationGraph) ─
    graphCheckpoint,

    // ── findings checkpoint (from input graph fixture, not production path) ─
    findingsCount: {
      total: (params?.graph && Array.isArray(params.graph))
        ? params.graph.length
        : (params?.graph && params.graph.nodes != null)
          ? params.graph.nodes.length
          : 0,
      eligible: "UNAVAILABLE", // not canonically observable without production contract change
    },

    // ── CU narrative (NOT captured — synthesis is a separate seam from updateCase()) ─
    currentUnderstanding: "UNAVAILABLE",

    // ── candidate evidence (UNAVAILABLE without changing production code) ────────
    eligibleFindingPropositions: "UNAVAILABLE",

    // ── helper internals (for diagnostic / test verification) ────────────────────
    selectedQuestion: result?.proposal?.selectedQuestion ?? null,
    behaviourSelection: result?.behaviourSelection ?? null,
    reasoningState: _extractReasoningState(result),
  };
}

/**
 * Minimal deterministic category derivation from free-text (mirrors the production pipeline).
 */
function _deriveCategory(text) {
  if (!text || !text.trim()) return "none";

  const lower = text.toLowerCase();

  // Conditional patterns
  if (/normally\s+(?:avoid|skip|not\s+take|would\s+n't|can\'t)/i.test(lower) &&
      /(might|could|would\s+(?:accept|allow|take|do))/i.test(lower)) {
    return "conditional_tradeoff";
  }

  // Contrast / but patterns
  if (/but\s+i?\s*(don\'t|cannot|can\'t|won\'t|will\s+not)/i.test(lower)) {
    return "qualified_support";
  }

  // Explicit hard constraint
  if (/(hard\s+constraint|must\s+(?:not|never|always)|can\'?\s*t(?:o)\s*(?:not|be\s+able\s+to)|absolutely\s+cannot)/i.test(lower)) {
    return "hard_constraint";
  }

  // Strong preference / must positive
  if (/(must\s+(?:have|do|get)|absolutely\s+(?:need|require)|cannot\s+proceed\s+without)/i.test(lower)) {
    return "strong_preference";
  }

  // Relative importance
  if (/more\s+to\s+me|matters\s+more|higher\s+priority|top\s*priority/i.test(lower)) {
    return "relative_importance";
  }

  // Support / contraindicate
  if (/support(s)?\b|confirm(s)?\b|validates?\b/i.test(lower)) {
    return "supports_decision";
  }
  if (/contradicts?\b|against\s+it\b|i\s*don\'?\s*t\s*(?:think\s+so|agree)\b/i.test(lower)) {
    return "contradicts_decision";
  }

  return "cannot_determine";
}

/**
 * Extract reasoning state from a raw updateCase() result object.
 * Exported for deterministic test verification of preserved behaviour.
 */
function _extractReasoningState(result) {
  const rs = result?.reasoningState;
  if (!rs) return null;
  return {
    turnCount:     rs.turnCount      ?? null,
    health:        rs.health          ?? null,
    phase:         rs.phase           ?? null,
    progress:      rs.progress        ?? null,
    unknownStatuses: (rs.unknownStatuses || []).map(s => s.id + ":" + s.status),
  };
}

/**
 * Run one live experiment through the real updateCase() production path.
 */
async function runLiveExperiment({ graph, previousQuestion, answer }) {
  const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL");
  const model   = assertRequiredEnv("OLLAMA_MODEL");

  if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
    throw new Error(
      `Live experiment harness refuses to use localhost fallback. ` +
      `OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.`
    );
  }

  const { updateCase } = await import(
    "../../lib/graph/orchestrator.js"
  );

  const { getProvider } = await import("../../lib/llm/provider.js");
  const provider = getProvider();
  let modelElapsedMs = null;
  let providerGenerateCalls = 0;

  const timedProvider = {
    async generateReconstruction(prompt, modelName) {
      providerGenerateCalls++;
      const startedAt = Date.now();
      const response = await provider.generateReconstruction(prompt, modelName);
      modelElapsedMs = Date.now() - startedAt;
      return response;
    },
  };

  const endToEndStartedAt = Date.now();
  const result = await updateCase(
    { situationGraph: graph, previousQuestion, answer },
    { provider: timedProvider },
  );
  const endToEndElapsedMs = Date.now() - endToEndStartedAt;

  return extractLiveExperimentCheckpoint(result, { graph, answer });
}

/**
 * Compose the canonical post-episode deep-investigation checkpoint sequence:
 *   updateCase → synthesizeCurrentUnderstanding
 *
 * Mirrors the production flow in ScenarioForm.handleDoneForNowPromotion / executeEpisodeDone:
 *   1. Call the episode done server seam (POST /api/cases/update, episodeMode=true)
 *      which updates the SituationGraph and accumulates Findings
 *   2. Use the updated graph + accumulated findings to synthesize Current Understanding
 *      via the same seam as production (POST /api/cases/synthesis → synthesizeCurrentUnderstanding)
 *
 * Returns a combined checkpoint containing:
 *   - extractLiveExperimentCheckpoint(...) fields from the episode done result
 *   - currentUnderstanding from the synthesis result
 *   - findingsCheckpoint with accurate post-episode accumulated findings count
 */
async function composeDeepInvestigationCheckpoint({ situationGraph, previousQuestion, answer }) {
  dotenv.config({ path: ".env.local" });

  const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL");
  const model   = assertRequiredEnv("OLLAMA_MODEL");

  if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
    throw new Error(
      `Live experiment harness refuses to use localhost fallback. ` +
      `OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.`
    );
  }

  /* ── Step 1: episode done (mirrors executeEpisodeDone server seam) ─── */
  const { updateCase } = await import(
    "../../lib/graph/orchestrator.js"
  );
  const { getProvider } = await import("../../lib/llm/provider.js");

  const provider = getProvider();
  let modelElapsedMs = null;
  let providerGenerateCalls = 0;

  const timedProvider = {
    async generateReconstruction(prompt, modelName) {
      providerGenerateCalls++;
      const startedAt = Date.now();
      const response = await provider.generateReconstruction(prompt, modelName);
      modelElapsedMs = Date.now() - startedAt;
      return response;
    },
  };

  const endToEndStartedAt = Date.now();
  const episodeResult = await updateCase(
    { situationGraph, previousQuestion, answer },
    { provider: timedProvider },
  );
  const endToEndElapsedMs = Date.now() - endToEndStartedAt;

  /* ── Step 2: Current Understanding synthesis (mirrors production synthesizeFn) ─── */
  const updatedGraph   = episodeResult.updatedSituationGraph;
  const accumulatedFindings = episodeResult.appendedFindings ?? [];

  // Collect accumulated findings from the episode result for accurate counting
  let postEpisodeFindings = Array.isArray(accumulatedFindings) ? [...accumulatedFindings] : [];

  // Derive eligible count via the same filter the production seam uses
  const { filterEligibleFindings } = await import("../../lib/graph/current-understanding-synthesis.js");
  const eligibleCount = postEpisodeFindings.length > 0
    ? filterEligibleFindings(postEpisodeFindings).length
    : 0;

  let currentUnderstanding = null;
  if (episodeResult.success && updatedGraph) {
    // Reuse the same provider/model plumbing — no re-creation, just reuse
    const { synthesizeCurrentUnderstanding } = await import("../../lib/graph/current-understanding-synthesis.js");

    const synthesized = await synthesizeCurrentUnderstanding(
      { situationGraph: updatedGraph, findings: postEpisodeFindings },
      { provider: timedProvider, modelName: model },
    );

    currentUnderstanding = synthesized?.currentUnderstanding ?? null;
  }

  return {
    /* ── CU synthesis result (NEW — mirrors production synthesizeFn) ─── */
    currentUnderstanding,

    /* ── Accurate post-episode findings count (FIX — uses actual accumulated Findings, not graph nodes) ─── */
    findingsCheckpoint: {
      total: postEpisodeFindings.length,
      eligible: eligibleCount,
    },

    /* ── Timing (mirrors extractLiveExperimentCheckpoint) ─── */
    modelElapsedMs,
    endToEndElapsedMs,
    providerGenerateCalls,
    modelName: episodeResult.modelName ?? null,

    /* ── Graph checkpoint (from production result) ─── */
    graphCheckpoint: updatedGraph
      ? {
          nodeCount:  updatedGraph.nodes?.length   ?? 0,
          edgeCount:  updatedGraph.edges?.length   ?? 0,
          resolvedNodeIds: updatedGraph.resolvedNodeIds ?? [],
        }
      : null,

    /* ── Answer meaning (from production result) ─── */
    userSupportedMeaning: episodeResult?.proposal?.answerMeaning?.userSupportedMeaning ?? null,
    possibleInference:    episodeResult?.proposal?.answerMeaning?.possibleInference     ?? null,
  };
}

module.exports = { runLiveExperiment, extractLiveExperimentCheckpoint, _extractReasoningState, composeDeepInvestigationCheckpoint };