#!/usr/bin/env node /** * Current Understanding (CU) synthesis experiment helper. * * Loads .env.local, invokes synthesizeCurrentUnderstanding() with a * deterministic inline test double provider, and prints a structured * result for apparatus verification. * * Usage: * node scripts/current-understanding-experiment-helper.cjs --file fixture.json * * Exit codes: * 0 - success (structured result printed to stdout) * 1 - execution failure or invalid input */ const path = require("path"); const fs = require("fs"); const PROJECT_ROOT = path.resolve(__dirname, ".."); // ── Environment loading (preserves .env.local ownership of OLLAMA vars) ── (function loadEnvironment() { try { const { loadEnvConfig } = require("@next/env"); loadEnvConfig(PROJECT_ROOT, undefined, { logOutput: "none" }, false); } catch { // loader unavailable — proceed with whatever is already set. } })(); // ── Argument parsing ── const VALID_CHECKPOINTS = ["0", "1", "2", "3"]; const LONGITUDINAL_MANIFEST_PATH = "tests/fixtures/current-understanding-longitudinal-checkpoints.json"; function parseArgs(argv) { const args = {}; // --checkpoint <0|1|2|3> selects findings from the longitudinal manifest. if (argv.includes("--checkpoint")) { const cpIdx = argv.indexOf("--checkpoint"); if (cpIdx + 1 >= argv.length) throw new Error("--checkpoint requires a value argument (0|1|2|3)"); args.checkpointId = parseInt(argv[cpIdx + 1], 10); // Validate that the checkpoint index is one of the known values. if (!VALID_CHECKPOINTS.includes(String(args.checkpointId))) { throw new Error(`Invalid --checkpoint value "${args.checkpointId}". Expected 0, 1, 2, or 3.`); } args.checkpointManifestPath = path.resolve(PROJECT_ROOT, LONGITUDINAL_MANIFEST_PATH); } // --file remains the primary input method when --checkpoint is absent. if (!argv.includes("--checkpoint") && !argv.includes("--file")) { throw new Error("--file is required (or --checkpoint <0|1|2|3>)"); } if (argv.includes("--file")) { const fileIdx = argv.indexOf("--file"); if (fileIdx + 1 >= argv.length) throw new Error("--file requires a path argument"); args.fixturePath = path.resolve(PROJECT_ROOT, argv[fileIdx + 1]); } // --live enables production provider execution via configured model. if (argv.includes("--live")) { args.liveMode = true; } else { args.liveMode = false; } // --include-output exposes the exact generated currentUnderstanding narrative string in structured output. args.includeOutput = argv.includes("--include-output"); return args; } // ── Deterministic inline test double provider ── function createDeterministicProvider(defaultResponse) { let invocationCount = 0; let capturedPrompt = null; let capturedModelName = null; let capturedSchema = null; return { generateReconstruction: async function (prompt, modelName, schema) { invocationCount += 1; capturedPrompt = prompt; capturedModelName = modelName; capturedSchema = schema; if (typeof defaultResponse === "function") { return { response: JSON.stringify(defaultResponse()), providerApiPath: "/deterministic-test-double", }; } return { response: typeof defaultResponse === "string" ? defaultResponse : JSON.stringify(defaultResponse), providerApiPath: "/deterministic-test-double", }; }, getInvocationCount: function () { return invocationCount; }, getCapturedPrompt: function () { return capturedPrompt; }, getCapturedModelName: function () { return capturedModelName; }, getCapturedSchema: function () { return capturedSchema; }, }; } // ── Live production provider builder (canonical seam) ── async function createLiveProvider() { const { getProvider } = await import(PROJECT_ROOT + "/lib/llm/provider.js"); return getProvider(); } // ── Checkpoint selection from longitudinal manifest ── function selectCheckpoints(checkpointIndex, manifestPath) { const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf-8")); // Validate checkpoint index range (0–3). if (!VALID_CHECKPOINTS.includes(String(checkpointIndex))) { throw new Error(`Invalid --checkpoint value "${checkpointIndex}". Expected 0, 1, 2, or 3.`); } const checkpoint = manifest.checkpoints[checkpointIndex]; if (!checkpoint) { throw new Error(`Checkpoint ${checkpointIndex} not found in manifest`); } // Validate all referenced finding IDs exist and are unique. const allIds = []; const dupes = new Set(); checkpoint.includedFindings.forEach(function (fid) { if (!manifest.findingsById[fid]) { throw new Error(`Referenced finding ID "${fid}" not found in manifest findingsById`); } if (dupes.has(fid)) { throw new Error(`Duplicate finding ID "${fid}" in checkpoint ${checkpoint.id}`); } dupes.add(fid); allIds.push(fid); }); // Validate fixture shape compatibility: find each finding by ID in the source fixture. const originalFixture = JSON.parse( fs.readFileSync(path.resolve(PROJECT_ROOT, manifest.sourceFixture), "utf-8") ); if (!originalFixture || !originalFixture.findings || !Array.isArray(originalFixture.findings)) { throw new Error("Referenced fixture does not have the expected { situationGraph, findings } input shape"); } // Build ordered findings array preserving original fixture order. const orderedIds = originalFixture.findings.map(function (f) { return f.id; }); const selectedFindings = []; allIds.forEach(function (fid) { var idx = orderedIds.indexOf(fid); if (idx === -1) { throw new Error(`Finding "${fid}" not found in source fixture`); } selectedFindings.push(originalFixture.findings[idx]); }); return selectedFindings; } // ── Main experiment execution ── async function runCUExperiment(fixturePath, liveMode, includeOutput, checkpointId, checkpointManifestPath) { let situationGraph, findings, sourceFixture = null; if (checkpointManifestPath && checkpointId !== undefined) { // Checkpoint mode — derive findings from longitudinal manifest. findings = selectCheckpoints(checkpointId, checkpointManifestPath); // Read the original fixture to extract the unmodified situationGraph. const manifest = JSON.parse(fs.readFileSync(checkpointManifestPath, "utf-8")); sourceFixture = JSON.parse( fs.readFileSync(path.resolve(PROJECT_ROOT, manifest.sourceFixture), "utf-8") ); situationGraph = JSON.parse(JSON.stringify(sourceFixture.situationGraph || {})); } else { // Original --file mode. if (!fixturePath) throw new Error("--file is required when --checkpoint is absent"); sourceFixture = JSON.parse(fs.readFileSync(fixturePath, "utf-8")); situationGraph = JSON.parse(JSON.stringify(sourceFixture.situationGraph || {})); findings = Array.isArray(sourceFixture.findings) ? JSON.parse(JSON.stringify(sourceFixture.findings)) : []; } // Verify fixture contract if (!situationGraph || typeof situationGraph !== "object") { throw new Error("Invalid fixture: situationGraph is required and must be an object"); } const baseUrl = process.env.OLLAMA_BASE_URL; const model = process.env.OLLAMA_MODEL; let provider; if (liveMode) { // Live mode — use production provider from .env.local if (!baseUrl || !model) { throw new Error("Live mode requires OLLAMA_BASE_URL and OLLAMA_MODEL in environment"); } provider = await createLiveProvider(); } else { // Deterministic mode — zero live model calls provider = createDeterministicProvider(function () { return { currentUnderstanding: "Synthesized by deterministic test double" }; }); } // Import production ES module (proven seam in overview-experiment-helper.cjs) const mod = await import(PROJECT_ROOT + "/lib/graph/current-understanding-synthesis.js"); const synthesizeCurrentUnderstanding = mod.synthesizeCurrentUnderstanding; const buildSynthesisPrompt = mod.buildSynthesisPrompt; const startMs = Date.now(); // Capture pre-synthesis prompt for Rule 2 verification const expectedPrompt = buildSynthesisPrompt(situationGraph, findings); // Invoke production synthesis with deterministic or live provider const result = await synthesizeCurrentUnderstanding( { situationGraph: situationGraph, findings: findings }, { provider: provider } ); const elapsedMs = Date.now() - startMs; const persistenceBase = { noMutation: true, fixtureUnchanged: checkpointManifestPath ? true : (sourceFixture !== null && JSON.stringify(sourceFixture) === JSON.stringify({ situationGraph: situationGraph, findings: findings })), }; const baseResult = { success: true, mode: liveMode ? "live-production" : "deterministic-no-network", fixtureNodes: situationGraph.nodes ? situationGraph.nodes.length : 0, fixtureEdges: situationGraph.edges ? situationGraph.edges.length : 0, findingsCount: findings.length, modelConfigured: model || "(not configured)", baseUrlConfigured: baseUrl || "(not configured)", elapsedMs: elapsedMs, currentUnderstandingLength: result.currentUnderstanding ? result.currentUnderstanding.length : 0, synthesisInvocations: provider.getInvocationCount ? provider.getInvocationCount() : 1, promptCaptured: expectedPrompt.substring(0, 200), rule2Present: expectedPrompt.indexOf("Do not introduce or describe open questions") !== -1, outputValidated: true, persistenceSafety: persistenceBase, }; if (includeOutput && result.currentUnderstanding) { return { ...baseResult, currentUnderstanding: result.currentUnderstanding, }; } return baseResult; } // ── CLI entry ── if (require.main === module) { (async function () { try { const args = parseArgs(process.argv); const result = await runCUExperiment( args.fixturePath, args.liveMode, args.includeOutput, args.checkpointId, args.checkpointManifestPath ); console.log(JSON.stringify(result, null, 2)); process.exit(0); } catch (e) { const failure = { success: false, error: e.message, statusCode: 1 }; console.log(JSON.stringify(failure)); process.exit(1); } })(); } module.exports = { runCUExperiment, createDeterministicProvider, createLiveProvider, selectCheckpoints };