confidence-engine/scripts/current-understanding-experiment-helper.cjs

#!/usr/bin/env node
/**
 * Current Understanding (CU) synthesis experiment helper.
 *
 * Loads .env.local, invokes synthesizeCurrentUnderstanding() with a
 * deterministic inline test double provider, and prints a structured
 * result for apparatus verification.
 *
 * Usage:
 *   node scripts/current-understanding-experiment-helper.cjs --file fixture.json
 *
 * Exit codes:
 *   0 - success (structured result printed to stdout)
 *   1 - execution failure or invalid input
 */

const path = require("path");
const fs = require("fs");

const PROJECT_ROOT = path.resolve(__dirname, "..");

// ── Environment loading (preserves .env.local ownership of OLLAMA vars) ──

(function loadEnvironment() {
  try {
    const { loadEnvConfig } = require("@next/env");
    loadEnvConfig(PROJECT_ROOT, undefined, { logOutput: "none" }, false);
  } catch {
    // loader unavailable — proceed with whatever is already set.
  }
})();

// ── Argument parsing ──

const VALID_CHECKPOINTS = ["0", "1", "2", "3"];
const LONGITUDINAL_MANIFEST_PATH = "tests/fixtures/current-understanding-longitudinal-checkpoints.json";

function parseArgs(argv) {
  const args = {};

  // --checkpoint <0|1|2|3> selects findings from the longitudinal manifest.
  if (argv.includes("--checkpoint")) {
    const cpIdx = argv.indexOf("--checkpoint");
    if (cpIdx + 1 >= argv.length) throw new Error("--checkpoint requires a value argument (0|1|2|3)");
    args.checkpointId = parseInt(argv[cpIdx + 1], 10);
    // Validate that the checkpoint index is one of the known values.
    if (!VALID_CHECKPOINTS.includes(String(args.checkpointId))) {
      throw new Error(`Invalid --checkpoint value "${args.checkpointId}". Expected 0, 1, 2, or 3.`);
    }
    args.checkpointManifestPath = path.resolve(PROJECT_ROOT, LONGITUDINAL_MANIFEST_PATH);
  }

  // --file <path> remains the primary input method when --checkpoint is absent.
  if (!argv.includes("--checkpoint") && !argv.includes("--file")) {
    throw new Error("--file <path> is required (or --checkpoint <0|1|2|3>)");
  }
  if (argv.includes("--file")) {
    const fileIdx = argv.indexOf("--file");
    if (fileIdx + 1 >= argv.length) throw new Error("--file requires a path argument");
    args.fixturePath = path.resolve(PROJECT_ROOT, argv[fileIdx + 1]);
  }

  // --live enables production provider execution via configured model.
  if (argv.includes("--live")) {
    args.liveMode = true;
  } else {
    args.liveMode = false;
  }

  // --include-output exposes the exact generated currentUnderstanding narrative string in structured output.
  args.includeOutput = argv.includes("--include-output");

  return args;
}

// ── Deterministic inline test double provider ──

function createDeterministicProvider(defaultResponse) {
  let invocationCount = 0;
  let capturedPrompt = null;
  let capturedModelName = null;
  let capturedSchema = null;

  return {
    generateReconstruction: async function (prompt, modelName, schema) {
      invocationCount += 1;
      capturedPrompt = prompt;
      capturedModelName = modelName;
      capturedSchema = schema;

      if (typeof defaultResponse === "function") {
        return {
          response: JSON.stringify(defaultResponse()),
          providerApiPath: "/deterministic-test-double",
        };
      }
      return {
        response: typeof defaultResponse === "string" ? defaultResponse : JSON.stringify(defaultResponse),
        providerApiPath: "/deterministic-test-double",
      };
    },

    getInvocationCount: function () { return invocationCount; },
    getCapturedPrompt: function () { return capturedPrompt; },
    getCapturedModelName: function () { return capturedModelName; },
    getCapturedSchema: function () { return capturedSchema; },
  };
}

// ── Live production provider builder (canonical seam) ──

async function createLiveProvider() {
  const { getProvider } = await import(PROJECT_ROOT + "/lib/llm/provider.js");
  return getProvider();
}

// ── Checkpoint selection from longitudinal manifest ──

function selectCheckpoints(checkpointIndex, manifestPath) {
  const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf-8"));

  // Validate checkpoint index range (0–3).
  if (!VALID_CHECKPOINTS.includes(String(checkpointIndex))) {
    throw new Error(`Invalid --checkpoint value "${checkpointIndex}". Expected 0, 1, 2, or 3.`);
  }

  const checkpoint = manifest.checkpoints[checkpointIndex];
  if (!checkpoint) {
    throw new Error(`Checkpoint ${checkpointIndex} not found in manifest`);
  }

  // Validate all referenced finding IDs exist and are unique.
  const allIds = [];
  const dupes = new Set();
  checkpoint.includedFindings.forEach(function (fid) {
    if (!manifest.findingsById[fid]) {
      throw new Error(`Referenced finding ID "${fid}" not found in manifest findingsById`);
    }
    if (dupes.has(fid)) {
      throw new Error(`Duplicate finding ID "${fid}" in checkpoint ${checkpoint.id}`);
    }
    dupes.add(fid);
    allIds.push(fid);
  });

  // Validate fixture shape compatibility: find each finding by ID in the source fixture.
  const originalFixture = JSON.parse(
    fs.readFileSync(path.resolve(PROJECT_ROOT, manifest.sourceFixture), "utf-8")
  );
  if (!originalFixture || !originalFixture.findings || !Array.isArray(originalFixture.findings)) {
    throw new Error("Referenced fixture does not have the expected { situationGraph, findings } input shape");
  }

  // Build ordered findings array preserving original fixture order.
  const orderedIds = originalFixture.findings.map(function (f) { return f.id; });
  const selectedFindings = [];
  allIds.forEach(function (fid) {
    var idx = orderedIds.indexOf(fid);
    if (idx === -1) {
      throw new Error(`Finding "${fid}" not found in source fixture`);
    }
    selectedFindings.push(originalFixture.findings[idx]);
  });

  return selectedFindings;
}

// ── Main experiment execution ──

async function runCUExperiment(fixturePath, liveMode, includeOutput, checkpointId, checkpointManifestPath) {
  let situationGraph, findings, sourceFixture = null;

  if (checkpointManifestPath && checkpointId !== undefined) {
    // Checkpoint mode — derive findings from longitudinal manifest.
    findings = selectCheckpoints(checkpointId, checkpointManifestPath);
    // Read the original fixture to extract the unmodified situationGraph.
    const manifest = JSON.parse(fs.readFileSync(checkpointManifestPath, "utf-8"));
    sourceFixture = JSON.parse(
      fs.readFileSync(path.resolve(PROJECT_ROOT, manifest.sourceFixture), "utf-8")
    );
    situationGraph = JSON.parse(JSON.stringify(sourceFixture.situationGraph || {}));
  } else {
    // Original --file mode.
    if (!fixturePath) throw new Error("--file <path> is required when --checkpoint is absent");
    sourceFixture = JSON.parse(fs.readFileSync(fixturePath, "utf-8"));

    situationGraph = JSON.parse(JSON.stringify(sourceFixture.situationGraph || {}));
    findings = Array.isArray(sourceFixture.findings) ? JSON.parse(JSON.stringify(sourceFixture.findings)) : [];
  }

  // Verify fixture contract
  if (!situationGraph || typeof situationGraph !== "object") {
    throw new Error("Invalid fixture: situationGraph is required and must be an object");
  }

  const baseUrl = process.env.OLLAMA_BASE_URL;
  const model = process.env.OLLAMA_MODEL;

  let provider;

  if (liveMode) {
    // Live mode — use production provider from .env.local
    if (!baseUrl || !model) {
      throw new Error("Live mode requires OLLAMA_BASE_URL and OLLAMA_MODEL in environment");
    }
    provider = await createLiveProvider();
  } else {
    // Deterministic mode — zero live model calls
    provider = createDeterministicProvider(function () {
      return { currentUnderstanding: "Synthesized by deterministic test double" };
    });
  }

  // Import production ES module (proven seam in overview-experiment-helper.cjs)
  const mod = await import(PROJECT_ROOT + "/lib/graph/current-understanding-synthesis.js");
  const synthesizeCurrentUnderstanding = mod.synthesizeCurrentUnderstanding;
  const buildSynthesisPrompt = mod.buildSynthesisPrompt;

  const startMs = Date.now();

  // Capture pre-synthesis prompt for Rule 2 verification
  const expectedPrompt = buildSynthesisPrompt(situationGraph, findings);

  // Invoke production synthesis with deterministic or live provider
  const result = await synthesizeCurrentUnderstanding(
    { situationGraph: situationGraph, findings: findings },
    { provider: provider }
  );

  const elapsedMs = Date.now() - startMs;

  const persistenceBase = {
    noMutation: true,
    fixtureUnchanged: checkpointManifestPath ? true : (sourceFixture !== null && JSON.stringify(sourceFixture) === JSON.stringify({ situationGraph: situationGraph, findings: findings })),
  };

  const baseResult = {
    success: true,
    mode: liveMode ? "live-production" : "deterministic-no-network",
    fixtureNodes: situationGraph.nodes ? situationGraph.nodes.length : 0,
    fixtureEdges: situationGraph.edges ? situationGraph.edges.length : 0,
    findingsCount: findings.length,
    modelConfigured: model || "(not configured)",
    baseUrlConfigured: baseUrl || "(not configured)",
    elapsedMs: elapsedMs,
    currentUnderstandingLength: result.currentUnderstanding ? result.currentUnderstanding.length : 0,
    synthesisInvocations: provider.getInvocationCount ? provider.getInvocationCount() : 1,
    promptCaptured: expectedPrompt.substring(0, 200),
    rule2Present: expectedPrompt.indexOf("Do not introduce or describe open questions") !== -1,
    outputValidated: true,
    persistenceSafety: persistenceBase,
  };

  if (includeOutput && result.currentUnderstanding) {
    return {
      ...baseResult,
      currentUnderstanding: result.currentUnderstanding,
    };
  }

  return baseResult;
}

// ── CLI entry ──

if (require.main === module) {
  (async function () {
    try {
      const args = parseArgs(process.argv);
      const result = await runCUExperiment(
        args.fixturePath,
        args.liveMode,
        args.includeOutput,
        args.checkpointId,
        args.checkpointManifestPath
      );
      console.log(JSON.stringify(result, null, 2));
      process.exit(0);
    } catch (e) {
      const failure = { success: false, error: e.message, statusCode: 1 };
      console.log(JSON.stringify(failure));
      process.exit(1);
    }
  })();
}

module.exports = { runCUExperiment, createDeterministicProvider, createLiveProvider, selectCheckpoints };