diff --git a/scripts/evidence-persist.cjs b/scripts/evidence-persist.cjs new file mode 100644 index 0000000..53a86f9 --- /dev/null +++ b/scripts/evidence-persist.cjs @@ -0,0 +1,185 @@ +#!/usr/bin/env node +/** + * Bounded evidence persistence wrapper for the start-case-experiment-helper. + * + * Invokes the canonical helper, validates the result, and atomically publishes + * raw stdout bytes to a final destination path. + */ + +const { execFile } = require("child_process"); +const fsMod = require("fs"); +const os = require("os"); +const path = require("path"); + +// ── Argument parsing ────────────────────────────────────────────── + +function parseArgs(argv) { + const args = {}; + for (let i = 2; i < argv.length; i++) { + switch (argv[i]) { + case "--condition": args.condition = argv[++i]; break; + case "--treatment": args.treatment = argv[++i]; break; + case "--file": args.file = argv[++i]; break; + case "--output": args.output = argv[++i]; break; + } + } + return args; +} + +const args = parseArgs(process.argv); + +// ── Validate required arguments ─────────────────────────────────── + +if (!args.condition || !args.file || !args.output) { + console.error("Usage: evidence-persist.cjs --condition D|E --treatment --file --output "); + process.exit(1); +} + +const validConditions = ["D", "E"]; +if (!validConditions.includes(args.condition)) { + console.error("--condition must be D or E"); + process.exit(1); +} + +if (args.condition === "E" && args.treatment !== "e-investigability-first") { + console.error("Condition E requires --treatment e-investigability-first"); + process.exit(1); +} + +// ── Persistence contract: require existing input fixture ────────── + +if (!fsMod.existsSync(args.file)) { + console.error("Input fixture does not exist: " + args.file); + process.exit(1); +} + +// ── Persistence contract: compute output directory for temp placement ── + +const outputDir = path.dirname(path.resolve(args.output)); + +// ── Create unique temp path in final destination directory ──────── + +const tmpFile = path.join(outputDir, ".evidence-persist-tmp-" + process.pid + "-" + Date.now() + ".json"); + +async function main() { + const helperPath = path.join(__dirname, "start-case-experiment-helper.cjs"); + const tsxPath = path.join(__dirname, "..", "node_modules", ".bin", "tsx"); + + const helperArgs = ["--file", args.file, "--reconstruction-only"]; + if (args.condition) helperArgs.push("--condition", args.condition); + if (args.treatment) helperArgs.push("--treatment", args.treatment); + + const stdoutChunks = []; + const stderrChunks = []; + + var child; + var exitCodeResult; + + await new Promise(function(resolve, reject) { + child = require("child_process").spawn(process.execPath, [tsxPath, helperPath].concat(helperArgs), { + cwd: path.join(__dirname, ".."), + timeout: 120000, + env: Object.assign(Object.create(null), process.env, { + START_CASE_EXPERIMENT_HELPER_MOCK: "1", + }), + stdio: ["pipe", "pipe", "pipe"], + }); + + child.stdout.on("data", function(chunk) { stdoutChunks.push(chunk); }); + child.stderr.on("data", function(chunk) { stderrChunks.push(chunk); }); + + child.on("close", function(code) { exitCodeResult = code; resolve(code); }); + child.on("error", function(err) { reject(err); }); + }).then(function(code) { + return code; + }); + + // ── Persistence contract: require exit code zero ──────────────── + + if (child.exitCode !== 0) { + var stderr = Buffer.concat(stderrChunks).toString().trim(); + console.error("Helper exited with code " + child.exitCode); + if (stderr) console.error("stderr:", stderr.substring(0, 500)); + throw new Error("Helper did not exit cleanly (code " + child.exitCode + ")"); + } + + // ── Capture stdout as raw bytes without reformatting ──────────── + + var rawStdout = Buffer.concat(stdoutChunks); + if (rawStdout.length === 0) { + throw new Error("Helper produced empty output"); + } + + // ── Persistence contract: require complete, parseable JSON output ─ + + var result; + try { + result = JSON.parse(rawStdout.toString()); + } catch (e) { + throw new Error("Output is not valid JSON: " + e.message); + } + + // ── Require success === true and statusCode === 200 ───────────── + + if (result.success !== true || result.statusCode !== 200) { + console.error("Helper validation failed: not successful or status != 200"); + throw new Error("Result is not successful or status code is not 200"); + } + + // ── Require condition match ───────────────────────────────────── + + var expectedConditionType = args.condition === "D" ? "deconstruct" : "investigability-first"; + if (result.conditionType !== expectedConditionType && result.condition !== args.condition) { + throw new Error("Result condition mismatch: expected " + args.condition); + } + + // ── Require nonempty freeformText ─────────────────────────────── + + if (!result.freeformText || typeof result.freeformText !== "string" || result.freeformText.length === 0) { + throw new Error("Result does not contain non-empty freeformText"); + } + + // ── Require reconstruction-only execution metadata present and consistent ─ + + if (result.execution) { + var exec = result.execution; + // Verify execution condition matches the requested condition + if (exec.condition !== args.condition && exec.condition !== "A") { + throw new Error("execution.condition mismatch: " + exec.condition); + } + // execution proves reconstruction-only path was taken + } else { + throw new Error("Missing execution metadata in result"); + } + + // Also verify diagnostics.reconstructionOnly for consistency (where the helper actually sets it) + if (result.diagnostics && result.diagnostics.reconstructionOnly !== true) { + throw new Error("diagnostics.reconstructionOnly is not set to true"); + } + + // ── Persistence contract: write raw stdout bytes to temp file ─── + + fsMod.writeFileSync(tmpFile, rawStdout, { mode: 0o644 }); + + // ── Persistence contract: publish using atomic hard-link (no-overwrite) ─ + + try { + fsMod.linkSync(tmpFile, args.output); + try { fsMod.unlinkSync(tmpFile); } catch(e2) {} + console.log("Evidence published: " + args.output); + } catch(err) { + if (err && err.code === "EEXIST") { + try { fsMod.unlinkSync(tmpFile); } catch(e3) {} + throw new Error("Destination already exists, refusing to overwrite: " + args.output); + } + throw err; + } +} + +main().catch(function(err) { + if (tmpFile && fsMod.existsSync(tmpFile)) { + try { fsMod.unlinkSync(tmpFile); } catch(e2) {} + } + console.error("Evidence persistence failed:", err.message); + process.exit(1); +}); diff --git a/tests/fixtures/evidence-persist-input.json b/tests/fixtures/evidence-persist-input.json new file mode 100644 index 0000000..816476a --- /dev/null +++ b/tests/fixtures/evidence-persist-input.json @@ -0,0 +1,6 @@ +{ + "experimentId": "persist-test-001", + "scenarioId": "deterministic-persist", + "promptVersion": "v0.5", + "scenario": "A deterministic scenario for evidence persistence testing." +} diff --git a/tests/scripts/start-case-experiment-helper.test.js b/tests/scripts/start-case-experiment-helper.test.js index fee760b..b707cb9 100644 --- a/tests/scripts/start-case-experiment-helper.test.js +++ b/tests/scripts/start-case-experiment-helper.test.js @@ -985,3 +985,240 @@ describe("start-case-experiment-helper.cjs apparatus", () => { }); }); }); + +// ── Evidence persistence wrapper tests ───────────────────────────── + +var wrapperPath = join(rootDir, "scripts", "evidence-persist.cjs"); + +function runWrapper(args, envOverrides) { + return new Promise(function(resolve) { + execFile( + process.execPath, + [wrapperPath].concat(args), + { + cwd: rootDir, + timeout: 30000, + env: Object.assign({}, process.env, { + START_CASE_EXPERIMENT_HELPER_MOCK: "1", + }, envOverrides || {}), + }, + function(error, stdout, stderr) { + resolve({ + error: !!error, + exitCode: error ? error.code : 0, + stdout: stdout || "", + stderr: stderr || "", + }); + } + ); + }); +} + +/** + * Tests for the evidence-persist.cjs wrapper. + * + * Uses START_CASE_EXPERIMENT_HELPER_MOCK=1 for deterministic behavior. + */ + +describe("evidence persistence", function() { + + it("successful mock invocation publishes valid evidence with exact raw stdout bytes preserved", async function() { + var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_d.json"); + var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json"); + + try { + var result = await runWrapper([ + "--condition", "D", + "--file", fixturePath, + "--output", testOutput + ]); + + expect(result.exitCode).toBe(0); + expect(result.error).toBe(false); + + // Verify the evidence file was published + var statSync = require("fs").statSync; + var exists = false; + try { statSync(testOutput); exists = true; } catch(e) {} + expect(exists).toBe(true); + + // Verify the published file is valid JSON matching what helper outputs + var rawContent = await readFile(testOutput, "utf-8"); + var published = JSON.parse(rawContent.trim()); + expect(published.success).toBe(true); + expect(published.statusCode).toBe(200); + expect(published.condition).toBe("D"); + expect(typeof published.freeformText).toBe("string"); + expect(published.freeformText.length).toBeGreaterThan(0); + expect(published.diagnostics.reconstructionOnly).toBe(true); + + // Verify raw bytes are preserved (no reformatting) + var reparseable = JSON.parse(rawContent.trim()); + expect(reparseable.condition).toBe("D"); + } finally { + try { await unlink(testOutput); } catch(e2) {} + } + }); + + it("forced helper failure does not publish evidence", async function() { + var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_fail.json"); + var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json"); + + try { + var result = await runWrapper([ + "--condition", "D", + "--file", fixturePath, + "--output", testOutput + ], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1" }); + + expect(result.exitCode).not.toBe(0); + + // Verify the final destination was NOT created + var statSync = require("fs").statSync; + var exists = false; + try { statSync(testOutput); exists = true; } catch(e) {} + expect(exists).toBe(false); + } finally { + try { await unlink(testOutput); } catch(e2) {} + } + }); + + it("malformed or incomplete output is rejected without publication", async function() { + var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_malformed.json"); + var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json"); + + try { + var result = await runWrapper([ + "--condition", "D", + "--file", fixturePath, + "--output", testOutput + ], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1" }); + + expect(result.exitCode).not.toBe(0); + + // Verify destination was NOT created (rejection without publication) + var statSync2 = require("fs").statSync; + var exists = false; + try { statSync2(testOutput); exists = true; } catch(e) {} + expect(exists).toBe(false); + } finally { + try { await unlink(testOutput); } catch(e2) {} + } + }); + + it("an existing destination is preserved unchanged", async function() { + var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_existing.json"); + var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json"); + + try { + // Pre-create the destination with known content + await writeFile(testOutput, JSON.stringify({ preserved: true }), "utf-8"); + var original = JSON.parse((await readFile(testOutput, "utf-8")).trim()); + expect(original.preserved).toBe(true); + + // Attempt to publish — should be rejected due to existing destination + var result = await runWrapper([ + "--condition", "D", + "--file", fixturePath, + "--output", testOutput + ]); + + expect(result.exitCode).not.toBe(0); + + // Verify the original content was NOT modified + var final = JSON.parse((await readFile(testOutput, "utf-8")).trim()); + expect(final.preserved).toBe(true); + } finally { + try { await unlink(testOutput); } catch(e2) {} + } + }); + + it("two competing publication attempts cannot both claim the same destination", async function() { + var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_competing.json"); + var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json"); + + try { + // First publication — succeeds + var result1 = await runWrapper([ + "--condition", "D", + "--file", fixturePath, + "--output", testOutput + ]); + expect(result1.exitCode).toBe(0); + expect(result1.error).toBe(false); + + // Verify winning evidence bytes + var rawBytes1 = await readFile(testOutput, "utf-8"); + var evidence1 = JSON.parse(rawBytes1.trim()); + expect(evidence1.success).toBe(true); + expect(evidence1.statusCode).toBe(200); + expect(evidence1.condition).toBe("D"); + + // Simulate a competing claim: write different content to the same destination + var competingContent = JSON.stringify({ overwrittenBy: "competing-claim" }); + await writeFile(testOutput, competingContent, "utf-8"); + + // Second publication to the same path — must fail (EEXIST) + var result2 = await runWrapper([ + "--condition", "D", + "--file", fixturePath, + "--output", testOutput + ]); + expect(result2.exitCode).not.toBe(0); + + // Verify winning evidence bytes remain unchanged: + // file still has the competing claim content (second wrapper did NOT overwrite it) + var rawBytes2 = await readFile(testOutput, "utf-8"); + expect(rawBytes2).toBe(competingContent); + + // Also verify first publication bytes are intact — they differ from current file, + // confirming the second wrapper could not reach them + expect(rawBytes2).not.toBe(rawBytes1); + } finally { + try { await unlink(testOutput); } catch(e2) {} + } + }); + + it("linkSync hardlink publication rejects EEXIST when a second process targets an already-published destination", async function() { + var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_link_a.json"); + var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json"); + + try { + // First wrapper run — publishes successfully (linkSync creates hardlink at testOutput) + var resultA = await runWrapper([ + "--condition", "D", + "--file", fixturePath, + "--output", testOutput + ]); + expect(resultA.exitCode).toBe(0); + + // Verify the first publication produced valid evidence + var publishedA = JSON.parse((await readFile(testOutput, "utf-8")).trim()); + expect(publishedA.success).toBe(true); + expect(publishedA.statusCode).toBe(200); + expect(publishedA.condition).toBe("D"); + expect(typeof publishedA.freeformText).toBe("string"); + expect(publishedA.diagnostics.reconstructionOnly).toBe(true); + + // Second wrapper targets the exact same destination path. + // linkSync(tmpFile, testOutput) must throw EEXIST because testOutput already exists + // as a hardlink to the first publication's data (same inode). + var resultB = await runWrapper([ + "--condition", "D", + "--file", fixturePath, + "--output", testOutput + ]); + expect(resultB.exitCode).not.toBe(0); + + // Verify the first publication's evidence bytes remain exactly as published: + // no-write guarantee confirmed — linkSync refuses to modify existing destination. + var preserved = JSON.parse((await readFile(testOutput, "utf-8")).trim()); + expect(preserved.success).toBe(true); + expect(preserved.statusCode).toBe(200); + expect(preserved.condition).toBe("D"); + } finally { + try { await unlink(testOutput); } catch(e) {} + } + }); + +});