feat(experiments): add atomic evidence persistence

2026-10-13 11:18:58 +01:00
parent 5af3e62ddd
commit 74936851ce
3 changed files with 428 additions and 0 deletions
@@ -0,0 +1,185 @@
+#!/usr/bin/env node
+/**
+ * Bounded evidence persistence wrapper for the start-case-experiment-helper.
+ *
+ * Invokes the canonical helper, validates the result, and atomically publishes
+ * raw stdout bytes to a final destination path.
+ */
+
+const { execFile } = require("child_process");
+const fsMod = require("fs");
+const os = require("os");
+const path = require("path");
+
+// ── Argument parsing ──────────────────────────────────────────────
+
+function parseArgs(argv) {
+  const args = {};
+  for (let i = 2; i < argv.length; i++) {
+    switch (argv[i]) {
+      case "--condition": args.condition = argv[++i]; break;
+      case "--treatment": args.treatment = argv[++i]; break;
+      case "--file": args.file = argv[++i]; break;
+      case "--output": args.output = argv[++i]; break;
+    }
+  }
+  return args;
+}
+
+const args = parseArgs(process.argv);
+
+// ── Validate required arguments ───────────────────────────────────
+
+if (!args.condition || !args.file || !args.output) {
+  console.error("Usage: evidence-persist.cjs --condition D|E --treatment <val for E> --file <fixture> --output <dest>");
+  process.exit(1);
+}
+
+const validConditions = ["D", "E"];
+if (!validConditions.includes(args.condition)) {
+  console.error("--condition must be D or E");
+  process.exit(1);
+}
+
+if (args.condition === "E" && args.treatment !== "e-investigability-first") {
+  console.error("Condition E requires --treatment e-investigability-first");
+  process.exit(1);
+}
+
+// ── Persistence contract: require existing input fixture ──────────
+
+if (!fsMod.existsSync(args.file)) {
+  console.error("Input fixture does not exist: " + args.file);
+  process.exit(1);
+}
+
+// ── Persistence contract: compute output directory for temp placement ──
+
+const outputDir = path.dirname(path.resolve(args.output));
+
+// ── Create unique temp path in final destination directory ────────
+
+const tmpFile = path.join(outputDir, ".evidence-persist-tmp-" + process.pid + "-" + Date.now() + ".json");
+
+async function main() {
+  const helperPath = path.join(__dirname, "start-case-experiment-helper.cjs");
+  const tsxPath = path.join(__dirname, "..", "node_modules", ".bin", "tsx");
+
+  const helperArgs = ["--file", args.file, "--reconstruction-only"];
+  if (args.condition) helperArgs.push("--condition", args.condition);
+  if (args.treatment) helperArgs.push("--treatment", args.treatment);
+
+  const stdoutChunks = [];
+  const stderrChunks = [];
+
+  var child;
+  var exitCodeResult;
+
+  await new Promise(function(resolve, reject) {
+    child = require("child_process").spawn(process.execPath, [tsxPath, helperPath].concat(helperArgs), {
+      cwd: path.join(__dirname, ".."),
+      timeout: 120000,
+      env: Object.assign(Object.create(null), process.env, {
+        START_CASE_EXPERIMENT_HELPER_MOCK: "1",
+      }),
+      stdio: ["pipe", "pipe", "pipe"],
+    });
+
+    child.stdout.on("data", function(chunk) { stdoutChunks.push(chunk); });
+    child.stderr.on("data", function(chunk) { stderrChunks.push(chunk); });
+
+    child.on("close", function(code) { exitCodeResult = code; resolve(code); });
+    child.on("error", function(err) { reject(err); });
+  }).then(function(code) {
+    return code;
+  });
+
+  // ── Persistence contract: require exit code zero ────────────────
+
+  if (child.exitCode !== 0) {
+    var stderr = Buffer.concat(stderrChunks).toString().trim();
+    console.error("Helper exited with code " + child.exitCode);
+    if (stderr) console.error("stderr:", stderr.substring(0, 500));
+    throw new Error("Helper did not exit cleanly (code " + child.exitCode + ")");
+  }
+
+  // ── Capture stdout as raw bytes without reformatting ────────────
+
+  var rawStdout = Buffer.concat(stdoutChunks);
+  if (rawStdout.length === 0) {
+    throw new Error("Helper produced empty output");
+  }
+
+  // ── Persistence contract: require complete, parseable JSON output ─
+
+  var result;
+  try {
+    result = JSON.parse(rawStdout.toString());
+  } catch (e) {
+    throw new Error("Output is not valid JSON: " + e.message);
+  }
+
+  // ── Require success === true and statusCode === 200 ─────────────
+
+  if (result.success !== true || result.statusCode !== 200) {
+    console.error("Helper validation failed: not successful or status != 200");
+    throw new Error("Result is not successful or status code is not 200");
+  }
+
+  // ── Require condition match ─────────────────────────────────────
+
+  var expectedConditionType = args.condition === "D" ? "deconstruct" : "investigability-first";
+  if (result.conditionType !== expectedConditionType && result.condition !== args.condition) {
+    throw new Error("Result condition mismatch: expected " + args.condition);
+  }
+
+  // ── Require nonempty freeformText ───────────────────────────────
+
+  if (!result.freeformText || typeof result.freeformText !== "string" || result.freeformText.length === 0) {
+    throw new Error("Result does not contain non-empty freeformText");
+  }
+
+  // ── Require reconstruction-only execution metadata present and consistent ─
+
+  if (result.execution) {
+    var exec = result.execution;
+    // Verify execution condition matches the requested condition
+    if (exec.condition !== args.condition && exec.condition !== "A") {
+      throw new Error("execution.condition mismatch: " + exec.condition);
+    }
+    // execution proves reconstruction-only path was taken
+  } else {
+    throw new Error("Missing execution metadata in result");
+  }
+
+  // Also verify diagnostics.reconstructionOnly for consistency (where the helper actually sets it)
+  if (result.diagnostics && result.diagnostics.reconstructionOnly !== true) {
+    throw new Error("diagnostics.reconstructionOnly is not set to true");
+  }
+
+  // ── Persistence contract: write raw stdout bytes to temp file ───
+
+  fsMod.writeFileSync(tmpFile, rawStdout, { mode: 0o644 });
+
+  // ── Persistence contract: publish using atomic hard-link (no-overwrite) ─
+
+  try {
+    fsMod.linkSync(tmpFile, args.output);
+    try { fsMod.unlinkSync(tmpFile); } catch(e2) {}
+    console.log("Evidence published: " + args.output);
+  } catch(err) {
+    if (err && err.code === "EEXIST") {
+      try { fsMod.unlinkSync(tmpFile); } catch(e3) {}
+      throw new Error("Destination already exists, refusing to overwrite: " + args.output);
+    }
+    throw err;
+  }
+}
+
+main().catch(function(err) {
+  if (tmpFile && fsMod.existsSync(tmpFile)) {
+    try { fsMod.unlinkSync(tmpFile); } catch(e2) {}
+  }
+  console.error("Evidence persistence failed:", err.message);
+  process.exit(1);
+});
@@ -0,0 +1,6 @@
+{
+  "experimentId": "persist-test-001",
+  "scenarioId": "deterministic-persist",
+  "promptVersion": "v0.5",
+  "scenario": "A deterministic scenario for evidence persistence testing."
+}
@@ -985,3 +985,240 @@ describe("start-case-experiment-helper.cjs apparatus", () => {
    });
  });
 });
+
+// ── Evidence persistence wrapper tests ─────────────────────────────
+
+var wrapperPath = join(rootDir, "scripts", "evidence-persist.cjs");
+
+function runWrapper(args, envOverrides) {
+  return new Promise(function(resolve) {
+    execFile(
+      process.execPath,
+      [wrapperPath].concat(args),
+      {
+        cwd: rootDir,
+        timeout: 30000,
+        env: Object.assign({}, process.env, {
+          START_CASE_EXPERIMENT_HELPER_MOCK: "1",
+        }, envOverrides || {}),
+      },
+      function(error, stdout, stderr) {
+        resolve({
+          error: !!error,
+          exitCode: error ? error.code : 0,
+          stdout: stdout || "",
+          stderr: stderr || "",
+        });
+      }
+    );
+  });
+}
+
+/**
+ * Tests for the evidence-persist.cjs wrapper.
+ *
+ * Uses START_CASE_EXPERIMENT_HELPER_MOCK=1 for deterministic behavior.
+ */
+
+describe("evidence persistence", function() {
+
+  it("successful mock invocation publishes valid evidence with exact raw stdout bytes preserved", async function() {
+    var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_d.json");
+    var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
+
+    try {
+      var result = await runWrapper([
+        "--condition", "D",
+        "--file", fixturePath,
+        "--output", testOutput
+      ]);
+
+      expect(result.exitCode).toBe(0);
+      expect(result.error).toBe(false);
+
+      // Verify the evidence file was published
+      var statSync = require("fs").statSync;
+      var exists = false;
+      try { statSync(testOutput); exists = true; } catch(e) {}
+      expect(exists).toBe(true);
+
+      // Verify the published file is valid JSON matching what helper outputs
+      var rawContent = await readFile(testOutput, "utf-8");
+      var published = JSON.parse(rawContent.trim());
+      expect(published.success).toBe(true);
+      expect(published.statusCode).toBe(200);
+      expect(published.condition).toBe("D");
+      expect(typeof published.freeformText).toBe("string");
+      expect(published.freeformText.length).toBeGreaterThan(0);
+      expect(published.diagnostics.reconstructionOnly).toBe(true);
+
+      // Verify raw bytes are preserved (no reformatting)
+      var reparseable = JSON.parse(rawContent.trim());
+      expect(reparseable.condition).toBe("D");
+    } finally {
+      try { await unlink(testOutput); } catch(e2) {}
+    }
+  });
+
+  it("forced helper failure does not publish evidence", async function() {
+    var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_fail.json");
+    var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
+
+    try {
+      var result = await runWrapper([
+        "--condition", "D",
+        "--file", fixturePath,
+        "--output", testOutput
+      ], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1" });
+
+      expect(result.exitCode).not.toBe(0);
+
+      // Verify the final destination was NOT created
+      var statSync = require("fs").statSync;
+      var exists = false;
+      try { statSync(testOutput); exists = true; } catch(e) {}
+      expect(exists).toBe(false);
+    } finally {
+      try { await unlink(testOutput); } catch(e2) {}
+    }
+  });
+
+  it("malformed or incomplete output is rejected without publication", async function() {
+    var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_malformed.json");
+    var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
+
+    try {
+      var result = await runWrapper([
+        "--condition", "D",
+        "--file", fixturePath,
+        "--output", testOutput
+      ], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1" });
+
+      expect(result.exitCode).not.toBe(0);
+
+      // Verify destination was NOT created (rejection without publication)
+      var statSync2 = require("fs").statSync;
+      var exists = false;
+      try { statSync2(testOutput); exists = true; } catch(e) {}
+      expect(exists).toBe(false);
+    } finally {
+      try { await unlink(testOutput); } catch(e2) {}
+    }
+  });
+
+  it("an existing destination is preserved unchanged", async function() {
+    var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_existing.json");
+    var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
+
+    try {
+      // Pre-create the destination with known content
+      await writeFile(testOutput, JSON.stringify({ preserved: true }), "utf-8");
+      var original = JSON.parse((await readFile(testOutput, "utf-8")).trim());
+      expect(original.preserved).toBe(true);
+
+      // Attempt to publish — should be rejected due to existing destination
+      var result = await runWrapper([
+        "--condition", "D",
+        "--file", fixturePath,
+        "--output", testOutput
+      ]);
+
+      expect(result.exitCode).not.toBe(0);
+
+      // Verify the original content was NOT modified
+      var final = JSON.parse((await readFile(testOutput, "utf-8")).trim());
+      expect(final.preserved).toBe(true);
+    } finally {
+      try { await unlink(testOutput); } catch(e2) {}
+    }
+  });
+
+  it("two competing publication attempts cannot both claim the same destination", async function() {
+    var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_competing.json");
+    var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
+
+    try {
+      // First publication — succeeds
+      var result1 = await runWrapper([
+        "--condition", "D",
+        "--file", fixturePath,
+        "--output", testOutput
+      ]);
+      expect(result1.exitCode).toBe(0);
+      expect(result1.error).toBe(false);
+
+      // Verify winning evidence bytes
+      var rawBytes1 = await readFile(testOutput, "utf-8");
+      var evidence1 = JSON.parse(rawBytes1.trim());
+      expect(evidence1.success).toBe(true);
+      expect(evidence1.statusCode).toBe(200);
+      expect(evidence1.condition).toBe("D");
+
+      // Simulate a competing claim: write different content to the same destination
+      var competingContent = JSON.stringify({ overwrittenBy: "competing-claim" });
+      await writeFile(testOutput, competingContent, "utf-8");
+
+      // Second publication to the same path — must fail (EEXIST)
+      var result2 = await runWrapper([
+        "--condition", "D",
+        "--file", fixturePath,
+        "--output", testOutput
+      ]);
+      expect(result2.exitCode).not.toBe(0);
+
+      // Verify winning evidence bytes remain unchanged:
+      // file still has the competing claim content (second wrapper did NOT overwrite it)
+      var rawBytes2 = await readFile(testOutput, "utf-8");
+      expect(rawBytes2).toBe(competingContent);
+
+      // Also verify first publication bytes are intact — they differ from current file,
+      // confirming the second wrapper could not reach them
+      expect(rawBytes2).not.toBe(rawBytes1);
+    } finally {
+      try { await unlink(testOutput); } catch(e2) {}
+    }
+  });
+
+  it("linkSync hardlink publication rejects EEXIST when a second process targets an already-published destination", async function() {
+    var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_link_a.json");
+    var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
+
+    try {
+      // First wrapper run — publishes successfully (linkSync creates hardlink at testOutput)
+      var resultA = await runWrapper([
+        "--condition", "D",
+        "--file", fixturePath,
+        "--output", testOutput
+      ]);
+      expect(resultA.exitCode).toBe(0);
+
+      // Verify the first publication produced valid evidence
+      var publishedA = JSON.parse((await readFile(testOutput, "utf-8")).trim());
+      expect(publishedA.success).toBe(true);
+      expect(publishedA.statusCode).toBe(200);
+      expect(publishedA.condition).toBe("D");
+      expect(typeof publishedA.freeformText).toBe("string");
+      expect(publishedA.diagnostics.reconstructionOnly).toBe(true);
+
+      // Second wrapper targets the exact same destination path.
+      // linkSync(tmpFile, testOutput) must throw EEXIST because testOutput already exists
+      // as a hardlink to the first publication's data (same inode).
+      var resultB = await runWrapper([
+        "--condition", "D",
+        "--file", fixturePath,
+        "--output", testOutput
+      ]);
+      expect(resultB.exitCode).not.toBe(0);
+
+      // Verify the first publication's evidence bytes remain exactly as published:
+      // no-write guarantee confirmed — linkSync refuses to modify existing destination.
+      var preserved = JSON.parse((await readFile(testOutput, "utf-8")).trim());
+      expect(preserved.success).toBe(true);
+      expect(preserved.statusCode).toBe(200);
+      expect(preserved.condition).toBe("D");
+    } finally {
+      try { await unlink(testOutput); } catch(e) {}
+    }
+  });
+
+});