confidence-engine/tests/graph/live-update-experiment-helper.cjs

/**
 * Canonical live-update experiment harness.
 *
 * Loads .env.local, validates Ollama configuration, invokes updateCase()
 * through the real production path, and captures the standard reasoning
 * checkpoints needed by reasoning experiments.
 *
 * Usage (from a test file):
 *   const { runLiveExperiment } = require("./tests/graph/live-update-experiment-helper.cjs");
 *
 *   const result = await runLiveExperiment({
 *     graph: /* SituationGraph fixture *\/,
 *     previousQuestion: "Is risk a hard constraint?",
 *     answer: "Risk matters more to me.",
 *   });
 */

const dotenv = require("dotenv");
dotenv.config({ path: ".env.local" });

function assertRequiredEnv(name) {
  const value = process.env[name];
  if (!value) {
    throw new Error(
      `Live experiment requires ${name}. Set it in .env.local.\n` +
      `Found: OLLAMA_BASE_URL=${process.env.OLLAMA_BASE_URL ?? "(missing)"}, ` +
      `OLLAMA_MODEL=${process.env.OLLAMA_MODEL ?? "(missing)"}`
    );
  }
  return value;
}

/**
 * Extract checkpoint data from a raw updateCase() result object.
 *
 * Pure function — no side effects, no I/O. Intended for use both by
 * `runLiveExperiment()` (apparatus wrapper) and by deterministic tests
 * that pass mock results directly.
 *
 * @param {object} result  — raw result from updateCase()
 * @param {object} params — { graph?, answer? }
 *   - graph: SituationGraph input fixture (for findingsCount total derived from input)
 *   - answer: previous question text for deriveCategory fallback
 * @returns {object} checkpoint object per spec contract
 */
function extractLiveExperimentCheckpoint(result, params) {
  const answerMeaning = result?.proposal?.answerMeaning || {};

  // Graph structural checkpoint (available from updatedSituationGraph only when present)
  const updatedGraph = result?.updatedSituationGraph;
  const graphCheckpoint = updatedGraph
    ? {
        nodeCount:       updatedGraph.nodes?.length      ?? 0,
        edgeCount:       updatedGraph.edges?.length      ?? 0,
        resolvedNodeIds: updatedGraph.resolvedNodeIds    ?? [],
      }
    : null;

  return {
    // ── timing (available via wrapper) ────────────────────
    modelElapsedMs:       result?.modelElapsedMs       !== undefined ? result.modelElapsedMs : null,
    endToEndElapsedMs:    result?.endToEndElapsedMs    !== undefined ? result.endToEndElapsedMs : null,
    providerGenerateCalls: result != null && result.providerGenerateCalls != null ? result.providerGenerateCalls : 0,
    modelName:            result?.modelName             || null,

    // ── answer meaning extraction ─────────────────────────
    userSupportedMeaning: answerMeaning.userSupportedMeaning ?? null,
    possibleInference:    answerMeaning.possibleInference    ?? null,
    rawAnswerCategory:    _deriveCategory(params?.answer),
    proposedMeaningCategory: _deriveCategory(answerMeaning.userSupportedMeaning || ""),

    // ── validation / guard checkpoints ───────────────────
    proposalValidation: result?.proposalValidation ?? { success: false, errors: [] },
    compatibilityGuard: result?.compatibilityApplied !== undefined
      ? { passed: result.compatibilityApplied, warnings: [], violations: [] }
      : { passed: false, warnings: [], violations: [] },

    // ── graph mutation checkpoint (from appliedGraph) ────
    graphMutation: result?.appliedGraph
      ? { nodes: result.appliedGraph.nodes ?? null, edges: result.appliedGraph.edges ?? null }
      : null,

    // ── graph structural checkpoint (available from updatedSituationGraph) ─
    graphCheckpoint,

    // ── findings checkpoint (from input graph fixture, not production path) ─
    findingsCount: {
      total: (params?.graph && Array.isArray(params.graph))
        ? params.graph.length
        : (params?.graph && params.graph.nodes != null)
          ? params.graph.nodes.length
          : 0,
      eligible: "UNAVAILABLE", // not canonically observable without production contract change
    },

    // ── CU narrative (NOT captured — synthesis is a separate seam from updateCase()) ─
    currentUnderstanding: "UNAVAILABLE",

    // ── candidate evidence (UNAVAILABLE without changing production code) ────────
    eligibleFindingPropositions: "UNAVAILABLE",

    // ── helper internals (for diagnostic / test verification) ────────────────────
    selectedQuestion: result?.proposal?.selectedQuestion ?? null,
    behaviourSelection: result?.behaviourSelection ?? null,
    reasoningState: _extractReasoningState(result),
  };
}

/**
 * Minimal deterministic category derivation from free-text (mirrors the production pipeline).
 */
function _deriveCategory(text) {
  if (!text || !text.trim()) return "none";

  const lower = text.toLowerCase();

  // Conditional patterns
  if (/normally\s+(?:avoid|skip|not\s+take|would\s+n't|can\'t)/i.test(lower) &&
      /(might|could|would\s+(?:accept|allow|take|do))/i.test(lower)) {
    return "conditional_tradeoff";
  }

  // Contrast / but patterns
  if (/but\s+i?\s*(don\'t|cannot|can\'t|won\'t|will\s+not)/i.test(lower)) {
    return "qualified_support";
  }

  // Explicit hard constraint
  if (/(hard\s+constraint|must\s+(?:not|never|always)|can\'?\s*t(?:o)\s*(?:not|be\s+able\s+to)|absolutely\s+cannot)/i.test(lower)) {
    return "hard_constraint";
  }

  // Strong preference / must positive
  if (/(must\s+(?:have|do|get)|absolutely\s+(?:need|require)|cannot\s+proceed\s+without)/i.test(lower)) {
    return "strong_preference";
  }

  // Relative importance
  if (/more\s+to\s+me|matters\s+more|higher\s+priority|top\s*priority/i.test(lower)) {
    return "relative_importance";
  }

  // Support / contraindicate
  if (/support(s)?\b|confirm(s)?\b|validates?\b/i.test(lower)) {
    return "supports_decision";
  }
  if (/contradicts?\b|against\s+it\b|i\s*don\'?\s*t\s*(?:think\s+so|agree)\b/i.test(lower)) {
    return "contradicts_decision";
  }

  return "cannot_determine";
}

/**
 * Extract reasoning state from a raw updateCase() result object.
 * Exported for deterministic test verification of preserved behaviour.
 */
function _extractReasoningState(result) {
  const rs = result?.reasoningState;
  if (!rs) return null;
  return {
    turnCount:     rs.turnCount      ?? null,
    health:        rs.health          ?? null,
    phase:         rs.phase           ?? null,
    progress:      rs.progress        ?? null,
    unknownStatuses: (rs.unknownStatuses || []).map(s => s.id + ":" + s.status),
  };
}

/**
 * Run one live experiment through the real updateCase() production path.
 */
async function runLiveExperiment({ graph, previousQuestion, answer }) {
  const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL");
  const model   = assertRequiredEnv("OLLAMA_MODEL");

  if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
    throw new Error(
      `Live experiment harness refuses to use localhost fallback. ` +
      `OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.`
    );
  }

  const { updateCase } = await import(
    "../../lib/graph/orchestrator.js"
  );

  const { getProvider } = await import("../../lib/llm/provider.js");
  const provider = getProvider();
  let modelElapsedMs = null;
  let providerGenerateCalls = 0;

  const timedProvider = {
    async generateReconstruction(prompt, modelName) {
      providerGenerateCalls++;
      const startedAt = Date.now();
      const response = await provider.generateReconstruction(prompt, modelName);
      modelElapsedMs = Date.now() - startedAt;
      return response;
    },
  };

  const endToEndStartedAt = Date.now();
  const result = await updateCase(
    { situationGraph: graph, previousQuestion, answer },
    { provider: timedProvider },
  );
  const endToEndElapsedMs = Date.now() - endToEndStartedAt;

  return extractLiveExperimentCheckpoint(result, { graph, answer });
}

module.exports = { runLiveExperiment, extractLiveExperimentCheckpoint, _extractReasoningState };