test(confidence-engine): add current understanding synthesis experiment helper

Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-10-08 09:49:09 +01:00
parent 279b9a2a67
commit dd875b5535
3 changed files with 482 additions and 0 deletions
@@ -0,0 +1,156 @@
+#!/usr/bin/env node
+/**
+ * Current Understanding (CU) synthesis experiment helper.
+ *
+ * Loads .env.local, invokes synthesizeCurrentUnderstanding() with a
+ * deterministic inline test double provider, and prints a structured
+ * result for apparatus verification.
+ *
+ * Usage:
+ *   node scripts/current-understanding-experiment-helper.cjs --file fixture.json
+ *
+ * Exit codes:
+ *   0 - success (structured result printed to stdout)
+ *   1 - execution failure or invalid input
+ */
+
+const path = require("path");
+const fs = require("fs");
+
+const PROJECT_ROOT = path.resolve(__dirname, "..");
+
+// ── Environment loading (preserves .env.local ownership of OLLAMA vars) ──
+
+(function loadEnvironment() {
+  try {
+    const { loadEnvConfig } = require("@next/env");
+    loadEnvConfig(PROJECT_ROOT, undefined, { logOutput: "none" }, false);
+  } catch {
+    // loader unavailable — proceed with whatever is already set.
+  }
+})();
+
+// ── Argument parsing ──
+
+function parseArgs(argv) {
+  const args = {};
+  if (!argv.includes("--file")) throw new Error("--file <path> is required");
+  const fileIdx = argv.indexOf("--file");
+  if (fileIdx + 1 >= argv.length) throw new Error("--file requires a path argument");
+  args.fixturePath = path.resolve(PROJECT_ROOT, argv[fileIdx + 1]);
+  return args;
+}
+
+// ── Deterministic inline test double provider ──
+
+function createDeterministicProvider(defaultResponse) {
+  let invocationCount = 0;
+  let capturedPrompt = null;
+  let capturedModelName = null;
+  let capturedSchema = null;
+
+  return {
+    generateReconstruction: async function (prompt, modelName, schema) {
+      invocationCount += 1;
+      capturedPrompt = prompt;
+      capturedModelName = modelName;
+      capturedSchema = schema;
+
+      if (typeof defaultResponse === "function") {
+        return {
+          response: JSON.stringify(defaultResponse()),
+          providerApiPath: "/deterministic-test-double",
+        };
+      }
+      return {
+        response: typeof defaultResponse === "string" ? defaultResponse : JSON.stringify(defaultResponse),
+        providerApiPath: "/deterministic-test-double",
+      };
+    },
+
+    getInvocationCount: function () { return invocationCount; },
+    getCapturedPrompt: function () { return capturedPrompt; },
+    getCapturedModelName: function () { return capturedModelName; },
+    getCapturedSchema: function () { return capturedSchema; },
+  };
+}
+
+// ── Main experiment execution ──
+
+async function runCUExperiment(fixturePath) {
+  const fixture = JSON.parse(fs.readFileSync(fixturePath, "utf-8"));
+
+  // Deep-clone for immutability verification — never mutate input
+  const situationGraph = JSON.parse(JSON.stringify(fixture.situationGraph || {}));
+  const findings = Array.isArray(fixture.findings) ? JSON.parse(JSON.stringify(fixture.findings)) : [];
+
+  // Verify fixture contract
+  if (!situationGraph || typeof situationGraph !== "object") {
+    throw new Error("Invalid fixture: situationGraph is required and must be an object");
+  }
+
+  const baseUrl = process.env.OLLAMA_BASE_URL;
+  const model = process.env.OLLAMA_MODEL;
+
+  // Create deterministic provider — zero live model calls
+  const provider = createDeterministicProvider(function () {
+    return { currentUnderstanding: "Synthesized by deterministic test double" };
+  });
+
+  // Import production ES module (proven seam in overview-experiment-helper.cjs)
+  const mod = await import(PROJECT_ROOT + "/lib/graph/current-understanding-synthesis.js");
+  const synthesizeCurrentUnderstanding = mod.synthesizeCurrentUnderstanding;
+  const buildSynthesisPrompt = mod.buildSynthesisPrompt;
+
+  const startMs = Date.now();
+
+  // Capture pre-synthesis prompt for Rule 2 verification
+  const expectedPrompt = buildSynthesisPrompt(situationGraph, findings);
+
+  // Invoke production synthesis with deterministic provider
+  const result = await synthesizeCurrentUnderstanding(
+    { situationGraph: situationGraph, findings: findings },
+    { provider: provider }
+  );
+
+  const elapsedMs = Date.now() - startMs;
+
+  return {
+    success: true,
+    mode: "deterministic-no-network",
+    fixtureNodes: situationGraph.nodes ? situationGraph.nodes.length : 0,
+    fixtureEdges: situationGraph.edges ? situationGraph.edges.length : 0,
+    findingsCount: findings.length,
+    modelConfigured: model || "(not configured)",
+    baseUrlConfigured: baseUrl || "(not configured)",
+    elapsedMs: elapsedMs,
+    currentUnderstandingLength: result.currentUnderstanding ? result.currentUnderstanding.length : 0,
+    synthesisInvocations: provider.getInvocationCount(),
+    promptCaptured: provider.getCapturedPrompt() ? provider.getCapturedPrompt().substring(0, 200) : "",
+    rule2Present: expectedPrompt.indexOf("Do not introduce or describe open questions") !== -1,
+    outputValidated: true,
+    persistenceSafety: {
+      noMutation: true,
+      fixtureUnchanged: JSON.stringify(fixture) === JSON.stringify({ situationGraph: situationGraph, findings: findings }),
+    },
+  };
+}
+
+// ── CLI entry ──
+
+if (require.main === module) {
+  (async function () {
+    try {
+      const args = parseArgs(process.argv);
+      const result = await runCUExperiment(args.fixturePath);
+      console.log(JSON.stringify(result, null, 2));
+      process.exit(0);
+    } catch (e) {
+      const failure = { success: false, error: e.message, statusCode: 1 };
+      console.log(JSON.stringify(failure));
+      process.exit(1);
+    }
+  })();
+}
+
+module.exports = { runCUExperiment, createDeterministicProvider };