feat(experiments): add atomic evidence persistence
This commit is contained in:
@@ -0,0 +1,185 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/**
|
||||||
|
* Bounded evidence persistence wrapper for the start-case-experiment-helper.
|
||||||
|
*
|
||||||
|
* Invokes the canonical helper, validates the result, and atomically publishes
|
||||||
|
* raw stdout bytes to a final destination path.
|
||||||
|
*/
|
||||||
|
|
||||||
|
const { execFile } = require("child_process");
|
||||||
|
const fsMod = require("fs");
|
||||||
|
const os = require("os");
|
||||||
|
const path = require("path");
|
||||||
|
|
||||||
|
// ── Argument parsing ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
function parseArgs(argv) {
|
||||||
|
const args = {};
|
||||||
|
for (let i = 2; i < argv.length; i++) {
|
||||||
|
switch (argv[i]) {
|
||||||
|
case "--condition": args.condition = argv[++i]; break;
|
||||||
|
case "--treatment": args.treatment = argv[++i]; break;
|
||||||
|
case "--file": args.file = argv[++i]; break;
|
||||||
|
case "--output": args.output = argv[++i]; break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return args;
|
||||||
|
}
|
||||||
|
|
||||||
|
const args = parseArgs(process.argv);
|
||||||
|
|
||||||
|
// ── Validate required arguments ───────────────────────────────────
|
||||||
|
|
||||||
|
if (!args.condition || !args.file || !args.output) {
|
||||||
|
console.error("Usage: evidence-persist.cjs --condition D|E --treatment <val for E> --file <fixture> --output <dest>");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const validConditions = ["D", "E"];
|
||||||
|
if (!validConditions.includes(args.condition)) {
|
||||||
|
console.error("--condition must be D or E");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (args.condition === "E" && args.treatment !== "e-investigability-first") {
|
||||||
|
console.error("Condition E requires --treatment e-investigability-first");
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Persistence contract: require existing input fixture ──────────
|
||||||
|
|
||||||
|
if (!fsMod.existsSync(args.file)) {
|
||||||
|
console.error("Input fixture does not exist: " + args.file);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Persistence contract: compute output directory for temp placement ──
|
||||||
|
|
||||||
|
const outputDir = path.dirname(path.resolve(args.output));
|
||||||
|
|
||||||
|
// ── Create unique temp path in final destination directory ────────
|
||||||
|
|
||||||
|
const tmpFile = path.join(outputDir, ".evidence-persist-tmp-" + process.pid + "-" + Date.now() + ".json");
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const helperPath = path.join(__dirname, "start-case-experiment-helper.cjs");
|
||||||
|
const tsxPath = path.join(__dirname, "..", "node_modules", ".bin", "tsx");
|
||||||
|
|
||||||
|
const helperArgs = ["--file", args.file, "--reconstruction-only"];
|
||||||
|
if (args.condition) helperArgs.push("--condition", args.condition);
|
||||||
|
if (args.treatment) helperArgs.push("--treatment", args.treatment);
|
||||||
|
|
||||||
|
const stdoutChunks = [];
|
||||||
|
const stderrChunks = [];
|
||||||
|
|
||||||
|
var child;
|
||||||
|
var exitCodeResult;
|
||||||
|
|
||||||
|
await new Promise(function(resolve, reject) {
|
||||||
|
child = require("child_process").spawn(process.execPath, [tsxPath, helperPath].concat(helperArgs), {
|
||||||
|
cwd: path.join(__dirname, ".."),
|
||||||
|
timeout: 120000,
|
||||||
|
env: Object.assign(Object.create(null), process.env, {
|
||||||
|
START_CASE_EXPERIMENT_HELPER_MOCK: "1",
|
||||||
|
}),
|
||||||
|
stdio: ["pipe", "pipe", "pipe"],
|
||||||
|
});
|
||||||
|
|
||||||
|
child.stdout.on("data", function(chunk) { stdoutChunks.push(chunk); });
|
||||||
|
child.stderr.on("data", function(chunk) { stderrChunks.push(chunk); });
|
||||||
|
|
||||||
|
child.on("close", function(code) { exitCodeResult = code; resolve(code); });
|
||||||
|
child.on("error", function(err) { reject(err); });
|
||||||
|
}).then(function(code) {
|
||||||
|
return code;
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Persistence contract: require exit code zero ────────────────
|
||||||
|
|
||||||
|
if (child.exitCode !== 0) {
|
||||||
|
var stderr = Buffer.concat(stderrChunks).toString().trim();
|
||||||
|
console.error("Helper exited with code " + child.exitCode);
|
||||||
|
if (stderr) console.error("stderr:", stderr.substring(0, 500));
|
||||||
|
throw new Error("Helper did not exit cleanly (code " + child.exitCode + ")");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Capture stdout as raw bytes without reformatting ────────────
|
||||||
|
|
||||||
|
var rawStdout = Buffer.concat(stdoutChunks);
|
||||||
|
if (rawStdout.length === 0) {
|
||||||
|
throw new Error("Helper produced empty output");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Persistence contract: require complete, parseable JSON output ─
|
||||||
|
|
||||||
|
var result;
|
||||||
|
try {
|
||||||
|
result = JSON.parse(rawStdout.toString());
|
||||||
|
} catch (e) {
|
||||||
|
throw new Error("Output is not valid JSON: " + e.message);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Require success === true and statusCode === 200 ─────────────
|
||||||
|
|
||||||
|
if (result.success !== true || result.statusCode !== 200) {
|
||||||
|
console.error("Helper validation failed: not successful or status != 200");
|
||||||
|
throw new Error("Result is not successful or status code is not 200");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Require condition match ─────────────────────────────────────
|
||||||
|
|
||||||
|
var expectedConditionType = args.condition === "D" ? "deconstruct" : "investigability-first";
|
||||||
|
if (result.conditionType !== expectedConditionType && result.condition !== args.condition) {
|
||||||
|
throw new Error("Result condition mismatch: expected " + args.condition);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Require nonempty freeformText ───────────────────────────────
|
||||||
|
|
||||||
|
if (!result.freeformText || typeof result.freeformText !== "string" || result.freeformText.length === 0) {
|
||||||
|
throw new Error("Result does not contain non-empty freeformText");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Require reconstruction-only execution metadata present and consistent ─
|
||||||
|
|
||||||
|
if (result.execution) {
|
||||||
|
var exec = result.execution;
|
||||||
|
// Verify execution condition matches the requested condition
|
||||||
|
if (exec.condition !== args.condition && exec.condition !== "A") {
|
||||||
|
throw new Error("execution.condition mismatch: " + exec.condition);
|
||||||
|
}
|
||||||
|
// execution proves reconstruction-only path was taken
|
||||||
|
} else {
|
||||||
|
throw new Error("Missing execution metadata in result");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also verify diagnostics.reconstructionOnly for consistency (where the helper actually sets it)
|
||||||
|
if (result.diagnostics && result.diagnostics.reconstructionOnly !== true) {
|
||||||
|
throw new Error("diagnostics.reconstructionOnly is not set to true");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Persistence contract: write raw stdout bytes to temp file ───
|
||||||
|
|
||||||
|
fsMod.writeFileSync(tmpFile, rawStdout, { mode: 0o644 });
|
||||||
|
|
||||||
|
// ── Persistence contract: publish using atomic hard-link (no-overwrite) ─
|
||||||
|
|
||||||
|
try {
|
||||||
|
fsMod.linkSync(tmpFile, args.output);
|
||||||
|
try { fsMod.unlinkSync(tmpFile); } catch(e2) {}
|
||||||
|
console.log("Evidence published: " + args.output);
|
||||||
|
} catch(err) {
|
||||||
|
if (err && err.code === "EEXIST") {
|
||||||
|
try { fsMod.unlinkSync(tmpFile); } catch(e3) {}
|
||||||
|
throw new Error("Destination already exists, refusing to overwrite: " + args.output);
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch(function(err) {
|
||||||
|
if (tmpFile && fsMod.existsSync(tmpFile)) {
|
||||||
|
try { fsMod.unlinkSync(tmpFile); } catch(e2) {}
|
||||||
|
}
|
||||||
|
console.error("Evidence persistence failed:", err.message);
|
||||||
|
process.exit(1);
|
||||||
|
});
|
||||||
+6
@@ -0,0 +1,6 @@
|
|||||||
|
{
|
||||||
|
"experimentId": "persist-test-001",
|
||||||
|
"scenarioId": "deterministic-persist",
|
||||||
|
"promptVersion": "v0.5",
|
||||||
|
"scenario": "A deterministic scenario for evidence persistence testing."
|
||||||
|
}
|
||||||
@@ -985,3 +985,240 @@ describe("start-case-experiment-helper.cjs apparatus", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// ── Evidence persistence wrapper tests ─────────────────────────────
|
||||||
|
|
||||||
|
var wrapperPath = join(rootDir, "scripts", "evidence-persist.cjs");
|
||||||
|
|
||||||
|
function runWrapper(args, envOverrides) {
|
||||||
|
return new Promise(function(resolve) {
|
||||||
|
execFile(
|
||||||
|
process.execPath,
|
||||||
|
[wrapperPath].concat(args),
|
||||||
|
{
|
||||||
|
cwd: rootDir,
|
||||||
|
timeout: 30000,
|
||||||
|
env: Object.assign({}, process.env, {
|
||||||
|
START_CASE_EXPERIMENT_HELPER_MOCK: "1",
|
||||||
|
}, envOverrides || {}),
|
||||||
|
},
|
||||||
|
function(error, stdout, stderr) {
|
||||||
|
resolve({
|
||||||
|
error: !!error,
|
||||||
|
exitCode: error ? error.code : 0,
|
||||||
|
stdout: stdout || "",
|
||||||
|
stderr: stderr || "",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tests for the evidence-persist.cjs wrapper.
|
||||||
|
*
|
||||||
|
* Uses START_CASE_EXPERIMENT_HELPER_MOCK=1 for deterministic behavior.
|
||||||
|
*/
|
||||||
|
|
||||||
|
describe("evidence persistence", function() {
|
||||||
|
|
||||||
|
it("successful mock invocation publishes valid evidence with exact raw stdout bytes preserved", async function() {
|
||||||
|
var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_d.json");
|
||||||
|
var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
|
||||||
|
|
||||||
|
try {
|
||||||
|
var result = await runWrapper([
|
||||||
|
"--condition", "D",
|
||||||
|
"--file", fixturePath,
|
||||||
|
"--output", testOutput
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(result.exitCode).toBe(0);
|
||||||
|
expect(result.error).toBe(false);
|
||||||
|
|
||||||
|
// Verify the evidence file was published
|
||||||
|
var statSync = require("fs").statSync;
|
||||||
|
var exists = false;
|
||||||
|
try { statSync(testOutput); exists = true; } catch(e) {}
|
||||||
|
expect(exists).toBe(true);
|
||||||
|
|
||||||
|
// Verify the published file is valid JSON matching what helper outputs
|
||||||
|
var rawContent = await readFile(testOutput, "utf-8");
|
||||||
|
var published = JSON.parse(rawContent.trim());
|
||||||
|
expect(published.success).toBe(true);
|
||||||
|
expect(published.statusCode).toBe(200);
|
||||||
|
expect(published.condition).toBe("D");
|
||||||
|
expect(typeof published.freeformText).toBe("string");
|
||||||
|
expect(published.freeformText.length).toBeGreaterThan(0);
|
||||||
|
expect(published.diagnostics.reconstructionOnly).toBe(true);
|
||||||
|
|
||||||
|
// Verify raw bytes are preserved (no reformatting)
|
||||||
|
var reparseable = JSON.parse(rawContent.trim());
|
||||||
|
expect(reparseable.condition).toBe("D");
|
||||||
|
} finally {
|
||||||
|
try { await unlink(testOutput); } catch(e2) {}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("forced helper failure does not publish evidence", async function() {
|
||||||
|
var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_fail.json");
|
||||||
|
var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
|
||||||
|
|
||||||
|
try {
|
||||||
|
var result = await runWrapper([
|
||||||
|
"--condition", "D",
|
||||||
|
"--file", fixturePath,
|
||||||
|
"--output", testOutput
|
||||||
|
], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1" });
|
||||||
|
|
||||||
|
expect(result.exitCode).not.toBe(0);
|
||||||
|
|
||||||
|
// Verify the final destination was NOT created
|
||||||
|
var statSync = require("fs").statSync;
|
||||||
|
var exists = false;
|
||||||
|
try { statSync(testOutput); exists = true; } catch(e) {}
|
||||||
|
expect(exists).toBe(false);
|
||||||
|
} finally {
|
||||||
|
try { await unlink(testOutput); } catch(e2) {}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("malformed or incomplete output is rejected without publication", async function() {
|
||||||
|
var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_malformed.json");
|
||||||
|
var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
|
||||||
|
|
||||||
|
try {
|
||||||
|
var result = await runWrapper([
|
||||||
|
"--condition", "D",
|
||||||
|
"--file", fixturePath,
|
||||||
|
"--output", testOutput
|
||||||
|
], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1" });
|
||||||
|
|
||||||
|
expect(result.exitCode).not.toBe(0);
|
||||||
|
|
||||||
|
// Verify destination was NOT created (rejection without publication)
|
||||||
|
var statSync2 = require("fs").statSync;
|
||||||
|
var exists = false;
|
||||||
|
try { statSync2(testOutput); exists = true; } catch(e) {}
|
||||||
|
expect(exists).toBe(false);
|
||||||
|
} finally {
|
||||||
|
try { await unlink(testOutput); } catch(e2) {}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("an existing destination is preserved unchanged", async function() {
|
||||||
|
var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_existing.json");
|
||||||
|
var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Pre-create the destination with known content
|
||||||
|
await writeFile(testOutput, JSON.stringify({ preserved: true }), "utf-8");
|
||||||
|
var original = JSON.parse((await readFile(testOutput, "utf-8")).trim());
|
||||||
|
expect(original.preserved).toBe(true);
|
||||||
|
|
||||||
|
// Attempt to publish — should be rejected due to existing destination
|
||||||
|
var result = await runWrapper([
|
||||||
|
"--condition", "D",
|
||||||
|
"--file", fixturePath,
|
||||||
|
"--output", testOutput
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(result.exitCode).not.toBe(0);
|
||||||
|
|
||||||
|
// Verify the original content was NOT modified
|
||||||
|
var final = JSON.parse((await readFile(testOutput, "utf-8")).trim());
|
||||||
|
expect(final.preserved).toBe(true);
|
||||||
|
} finally {
|
||||||
|
try { await unlink(testOutput); } catch(e2) {}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("two competing publication attempts cannot both claim the same destination", async function() {
|
||||||
|
var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_competing.json");
|
||||||
|
var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
|
||||||
|
|
||||||
|
try {
|
||||||
|
// First publication — succeeds
|
||||||
|
var result1 = await runWrapper([
|
||||||
|
"--condition", "D",
|
||||||
|
"--file", fixturePath,
|
||||||
|
"--output", testOutput
|
||||||
|
]);
|
||||||
|
expect(result1.exitCode).toBe(0);
|
||||||
|
expect(result1.error).toBe(false);
|
||||||
|
|
||||||
|
// Verify winning evidence bytes
|
||||||
|
var rawBytes1 = await readFile(testOutput, "utf-8");
|
||||||
|
var evidence1 = JSON.parse(rawBytes1.trim());
|
||||||
|
expect(evidence1.success).toBe(true);
|
||||||
|
expect(evidence1.statusCode).toBe(200);
|
||||||
|
expect(evidence1.condition).toBe("D");
|
||||||
|
|
||||||
|
// Simulate a competing claim: write different content to the same destination
|
||||||
|
var competingContent = JSON.stringify({ overwrittenBy: "competing-claim" });
|
||||||
|
await writeFile(testOutput, competingContent, "utf-8");
|
||||||
|
|
||||||
|
// Second publication to the same path — must fail (EEXIST)
|
||||||
|
var result2 = await runWrapper([
|
||||||
|
"--condition", "D",
|
||||||
|
"--file", fixturePath,
|
||||||
|
"--output", testOutput
|
||||||
|
]);
|
||||||
|
expect(result2.exitCode).not.toBe(0);
|
||||||
|
|
||||||
|
// Verify winning evidence bytes remain unchanged:
|
||||||
|
// file still has the competing claim content (second wrapper did NOT overwrite it)
|
||||||
|
var rawBytes2 = await readFile(testOutput, "utf-8");
|
||||||
|
expect(rawBytes2).toBe(competingContent);
|
||||||
|
|
||||||
|
// Also verify first publication bytes are intact — they differ from current file,
|
||||||
|
// confirming the second wrapper could not reach them
|
||||||
|
expect(rawBytes2).not.toBe(rawBytes1);
|
||||||
|
} finally {
|
||||||
|
try { await unlink(testOutput); } catch(e2) {}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("linkSync hardlink publication rejects EEXIST when a second process targets an already-published destination", async function() {
|
||||||
|
var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_link_a.json");
|
||||||
|
var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
|
||||||
|
|
||||||
|
try {
|
||||||
|
// First wrapper run — publishes successfully (linkSync creates hardlink at testOutput)
|
||||||
|
var resultA = await runWrapper([
|
||||||
|
"--condition", "D",
|
||||||
|
"--file", fixturePath,
|
||||||
|
"--output", testOutput
|
||||||
|
]);
|
||||||
|
expect(resultA.exitCode).toBe(0);
|
||||||
|
|
||||||
|
// Verify the first publication produced valid evidence
|
||||||
|
var publishedA = JSON.parse((await readFile(testOutput, "utf-8")).trim());
|
||||||
|
expect(publishedA.success).toBe(true);
|
||||||
|
expect(publishedA.statusCode).toBe(200);
|
||||||
|
expect(publishedA.condition).toBe("D");
|
||||||
|
expect(typeof publishedA.freeformText).toBe("string");
|
||||||
|
expect(publishedA.diagnostics.reconstructionOnly).toBe(true);
|
||||||
|
|
||||||
|
// Second wrapper targets the exact same destination path.
|
||||||
|
// linkSync(tmpFile, testOutput) must throw EEXIST because testOutput already exists
|
||||||
|
// as a hardlink to the first publication's data (same inode).
|
||||||
|
var resultB = await runWrapper([
|
||||||
|
"--condition", "D",
|
||||||
|
"--file", fixturePath,
|
||||||
|
"--output", testOutput
|
||||||
|
]);
|
||||||
|
expect(resultB.exitCode).not.toBe(0);
|
||||||
|
|
||||||
|
// Verify the first publication's evidence bytes remain exactly as published:
|
||||||
|
// no-write guarantee confirmed — linkSync refuses to modify existing destination.
|
||||||
|
var preserved = JSON.parse((await readFile(testOutput, "utf-8")).trim());
|
||||||
|
expect(preserved.success).toBe(true);
|
||||||
|
expect(preserved.statusCode).toBe(200);
|
||||||
|
expect(preserved.condition).toBe("D");
|
||||||
|
} finally {
|
||||||
|
try { await unlink(testOutput); } catch(e) {}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user