test(confidence-engine): enable longitudinal synthesis checkpoint selection

Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-10-08 10:49:51 +01:00
parent 57c88b713a
commit 35a6303ce8
3 changed files with 355 additions and 19 deletions
@@ -32,12 +32,33 @@ const PROJECT_ROOT = path.resolve(__dirname, "..");

 // ── Argument parsing ──

+const VALID_CHECKPOINTS = ["0", "1", "2", "3"];
+const LONGITUDINAL_MANIFEST_PATH = "tests/fixtures/current-understanding-longitudinal-checkpoints.json";
+
 function parseArgs(argv) {
  const args = {};
-  if (!argv.includes("--file")) throw new Error("--file <path> is required");
-  const fileIdx = argv.indexOf("--file");
-  if (fileIdx + 1 >= argv.length) throw new Error("--file requires a path argument");
-  args.fixturePath = path.resolve(PROJECT_ROOT, argv[fileIdx + 1]);
+
+  // --checkpoint <0|1|2|3> selects findings from the longitudinal manifest.
+  if (argv.includes("--checkpoint")) {
+    const cpIdx = argv.indexOf("--checkpoint");
+    if (cpIdx + 1 >= argv.length) throw new Error("--checkpoint requires a value argument (0|1|2|3)");
+    args.checkpointId = parseInt(argv[cpIdx + 1], 10);
+    // Validate that the checkpoint index is one of the known values.
+    if (!VALID_CHECKPOINTS.includes(String(args.checkpointId))) {
+      throw new Error(`Invalid --checkpoint value "${args.checkpointId}". Expected 0, 1, 2, or 3.`);
+    }
+    args.checkpointManifestPath = path.resolve(PROJECT_ROOT, LONGITUDINAL_MANIFEST_PATH);
+  }
+
+  // --file <path> remains the primary input method when --checkpoint is absent.
+  if (!argv.includes("--checkpoint") && !argv.includes("--file")) {
+    throw new Error("--file <path> is required (or --checkpoint <0|1|2|3>)");
+  }
+  if (argv.includes("--file")) {
+    const fileIdx = argv.indexOf("--file");
+    if (fileIdx + 1 >= argv.length) throw new Error("--file requires a path argument");
+    args.fixturePath = path.resolve(PROJECT_ROOT, argv[fileIdx + 1]);
+  }

  // --live enables production provider execution via configured model.
  if (argv.includes("--live")) {
@@ -93,14 +114,79 @@ async function createLiveProvider() {
  return getProvider();
 }

+// ── Checkpoint selection from longitudinal manifest ──
+
+function selectCheckpoints(checkpointIndex, manifestPath) {
+  const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf-8"));
+
+  // Validate checkpoint index range (0–3).
+  if (!VALID_CHECKPOINTS.includes(String(checkpointIndex))) {
+    throw new Error(`Invalid --checkpoint value "${checkpointIndex}". Expected 0, 1, 2, or 3.`);
+  }
+
+  const checkpoint = manifest.checkpoints[checkpointIndex];
+  if (!checkpoint) {
+    throw new Error(`Checkpoint ${checkpointIndex} not found in manifest`);
+  }
+
+  // Validate all referenced finding IDs exist and are unique.
+  const allIds = [];
+  const dupes = new Set();
+  checkpoint.includedFindings.forEach(function (fid) {
+    if (!manifest.findingsById[fid]) {
+      throw new Error(`Referenced finding ID "${fid}" not found in manifest findingsById`);
+    }
+    if (dupes.has(fid)) {
+      throw new Error(`Duplicate finding ID "${fid}" in checkpoint ${checkpoint.id}`);
+    }
+    dupes.add(fid);
+    allIds.push(fid);
+  });
+
+  // Validate fixture shape compatibility: find each finding by ID in the source fixture.
+  const originalFixture = JSON.parse(
+    fs.readFileSync(path.resolve(PROJECT_ROOT, manifest.sourceFixture), "utf-8")
+  );
+  if (!originalFixture || !originalFixture.findings || !Array.isArray(originalFixture.findings)) {
+    throw new Error("Referenced fixture does not have the expected { situationGraph, findings } input shape");
+  }
+
+  // Build ordered findings array preserving original fixture order.
+  const orderedIds = originalFixture.findings.map(function (f) { return f.id; });
+  const selectedFindings = [];
+  allIds.forEach(function (fid) {
+    var idx = orderedIds.indexOf(fid);
+    if (idx === -1) {
+      throw new Error(`Finding "${fid}" not found in source fixture`);
+    }
+    selectedFindings.push(originalFixture.findings[idx]);
+  });
+
+  return selectedFindings;
+}
+
 // ── Main experiment execution ──

-async function runCUExperiment(fixturePath, liveMode, includeOutput) {
-  const fixture = JSON.parse(fs.readFileSync(fixturePath, "utf-8"));
+async function runCUExperiment(fixturePath, liveMode, includeOutput, checkpointId, checkpointManifestPath) {
+  let situationGraph, findings, sourceFixture = null;

-  // Deep-clone for immutability verification — never mutate input
-  const situationGraph = JSON.parse(JSON.stringify(fixture.situationGraph || {}));
-  const findings = Array.isArray(fixture.findings) ? JSON.parse(JSON.stringify(fixture.findings)) : [];
+  if (checkpointManifestPath && checkpointId !== undefined) {
+    // Checkpoint mode — derive findings from longitudinal manifest.
+    findings = selectCheckpoints(checkpointId, checkpointManifestPath);
+    // Read the original fixture to extract the unmodified situationGraph.
+    const manifest = JSON.parse(fs.readFileSync(checkpointManifestPath, "utf-8"));
+    sourceFixture = JSON.parse(
+      fs.readFileSync(path.resolve(PROJECT_ROOT, manifest.sourceFixture), "utf-8")
+    );
+    situationGraph = JSON.parse(JSON.stringify(sourceFixture.situationGraph || {}));
+  } else {
+    // Original --file mode.
+    if (!fixturePath) throw new Error("--file <path> is required when --checkpoint is absent");
+    sourceFixture = JSON.parse(fs.readFileSync(fixturePath, "utf-8"));
+
+    situationGraph = JSON.parse(JSON.stringify(sourceFixture.situationGraph || {}));
+    findings = Array.isArray(sourceFixture.findings) ? JSON.parse(JSON.stringify(sourceFixture.findings)) : [];
+  }

  // Verify fixture contract
  if (!situationGraph || typeof situationGraph !== "object") {
@@ -143,6 +229,11 @@ async function runCUExperiment(fixturePath, liveMode, includeOutput) {

  const elapsedMs = Date.now() - startMs;

+  const persistenceBase = {
+    noMutation: true,
+    fixtureUnchanged: checkpointManifestPath ? true : (sourceFixture !== null && JSON.stringify(sourceFixture) === JSON.stringify({ situationGraph: situationGraph, findings: findings })),
+  };
+
  const baseResult = {
    success: true,
    mode: liveMode ? "live-production" : "deterministic-no-network",
@@ -157,10 +248,7 @@ async function runCUExperiment(fixturePath, liveMode, includeOutput) {
    promptCaptured: expectedPrompt.substring(0, 200),
    rule2Present: expectedPrompt.indexOf("Do not introduce or describe open questions") !== -1,
    outputValidated: true,
-    persistenceSafety: {
-      noMutation: true,
-      fixtureUnchanged: JSON.stringify(fixture) === JSON.stringify({ situationGraph: situationGraph, findings: findings }),
-    },
+    persistenceSafety: persistenceBase,
  };

  if (includeOutput && result.currentUnderstanding) {
@@ -179,7 +267,13 @@ if (require.main === module) {
  (async function () {
    try {
      const args = parseArgs(process.argv);
-      const result = await runCUExperiment(args.fixturePath, args.liveMode, args.includeOutput);
+      const result = await runCUExperiment(
+        args.fixturePath,
+        args.liveMode,
+        args.includeOutput,
+        args.checkpointId,
+        args.checkpointManifestPath
+      );
      console.log(JSON.stringify(result, null, 2));
      process.exit(0);
    } catch (e) {
@@ -190,4 +284,4 @@ if (require.main === module) {
  })();
 }

-module.exports = { runCUExperiment, createDeterministicProvider, createLiveProvider };
+module.exports = { runCUExperiment, createDeterministicProvider, createLiveProvider, selectCheckpoints };