feat(experiments): add atomic evidence persistence
This commit is contained in:
+6
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"experimentId": "persist-test-001",
|
||||
"scenarioId": "deterministic-persist",
|
||||
"promptVersion": "v0.5",
|
||||
"scenario": "A deterministic scenario for evidence persistence testing."
|
||||
}
|
||||
@@ -985,3 +985,240 @@ describe("start-case-experiment-helper.cjs apparatus", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ── Evidence persistence wrapper tests ─────────────────────────────
|
||||
|
||||
var wrapperPath = join(rootDir, "scripts", "evidence-persist.cjs");
|
||||
|
||||
function runWrapper(args, envOverrides) {
|
||||
return new Promise(function(resolve) {
|
||||
execFile(
|
||||
process.execPath,
|
||||
[wrapperPath].concat(args),
|
||||
{
|
||||
cwd: rootDir,
|
||||
timeout: 30000,
|
||||
env: Object.assign({}, process.env, {
|
||||
START_CASE_EXPERIMENT_HELPER_MOCK: "1",
|
||||
}, envOverrides || {}),
|
||||
},
|
||||
function(error, stdout, stderr) {
|
||||
resolve({
|
||||
error: !!error,
|
||||
exitCode: error ? error.code : 0,
|
||||
stdout: stdout || "",
|
||||
stderr: stderr || "",
|
||||
});
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests for the evidence-persist.cjs wrapper.
|
||||
*
|
||||
* Uses START_CASE_EXPERIMENT_HELPER_MOCK=1 for deterministic behavior.
|
||||
*/
|
||||
|
||||
describe("evidence persistence", function() {
|
||||
|
||||
it("successful mock invocation publishes valid evidence with exact raw stdout bytes preserved", async function() {
|
||||
var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_d.json");
|
||||
var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
|
||||
|
||||
try {
|
||||
var result = await runWrapper([
|
||||
"--condition", "D",
|
||||
"--file", fixturePath,
|
||||
"--output", testOutput
|
||||
]);
|
||||
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.error).toBe(false);
|
||||
|
||||
// Verify the evidence file was published
|
||||
var statSync = require("fs").statSync;
|
||||
var exists = false;
|
||||
try { statSync(testOutput); exists = true; } catch(e) {}
|
||||
expect(exists).toBe(true);
|
||||
|
||||
// Verify the published file is valid JSON matching what helper outputs
|
||||
var rawContent = await readFile(testOutput, "utf-8");
|
||||
var published = JSON.parse(rawContent.trim());
|
||||
expect(published.success).toBe(true);
|
||||
expect(published.statusCode).toBe(200);
|
||||
expect(published.condition).toBe("D");
|
||||
expect(typeof published.freeformText).toBe("string");
|
||||
expect(published.freeformText.length).toBeGreaterThan(0);
|
||||
expect(published.diagnostics.reconstructionOnly).toBe(true);
|
||||
|
||||
// Verify raw bytes are preserved (no reformatting)
|
||||
var reparseable = JSON.parse(rawContent.trim());
|
||||
expect(reparseable.condition).toBe("D");
|
||||
} finally {
|
||||
try { await unlink(testOutput); } catch(e2) {}
|
||||
}
|
||||
});
|
||||
|
||||
it("forced helper failure does not publish evidence", async function() {
|
||||
var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_fail.json");
|
||||
var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
|
||||
|
||||
try {
|
||||
var result = await runWrapper([
|
||||
"--condition", "D",
|
||||
"--file", fixturePath,
|
||||
"--output", testOutput
|
||||
], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1" });
|
||||
|
||||
expect(result.exitCode).not.toBe(0);
|
||||
|
||||
// Verify the final destination was NOT created
|
||||
var statSync = require("fs").statSync;
|
||||
var exists = false;
|
||||
try { statSync(testOutput); exists = true; } catch(e) {}
|
||||
expect(exists).toBe(false);
|
||||
} finally {
|
||||
try { await unlink(testOutput); } catch(e2) {}
|
||||
}
|
||||
});
|
||||
|
||||
it("malformed or incomplete output is rejected without publication", async function() {
|
||||
var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_malformed.json");
|
||||
var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
|
||||
|
||||
try {
|
||||
var result = await runWrapper([
|
||||
"--condition", "D",
|
||||
"--file", fixturePath,
|
||||
"--output", testOutput
|
||||
], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1" });
|
||||
|
||||
expect(result.exitCode).not.toBe(0);
|
||||
|
||||
// Verify destination was NOT created (rejection without publication)
|
||||
var statSync2 = require("fs").statSync;
|
||||
var exists = false;
|
||||
try { statSync2(testOutput); exists = true; } catch(e) {}
|
||||
expect(exists).toBe(false);
|
||||
} finally {
|
||||
try { await unlink(testOutput); } catch(e2) {}
|
||||
}
|
||||
});
|
||||
|
||||
it("an existing destination is preserved unchanged", async function() {
|
||||
var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_existing.json");
|
||||
var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
|
||||
|
||||
try {
|
||||
// Pre-create the destination with known content
|
||||
await writeFile(testOutput, JSON.stringify({ preserved: true }), "utf-8");
|
||||
var original = JSON.parse((await readFile(testOutput, "utf-8")).trim());
|
||||
expect(original.preserved).toBe(true);
|
||||
|
||||
// Attempt to publish — should be rejected due to existing destination
|
||||
var result = await runWrapper([
|
||||
"--condition", "D",
|
||||
"--file", fixturePath,
|
||||
"--output", testOutput
|
||||
]);
|
||||
|
||||
expect(result.exitCode).not.toBe(0);
|
||||
|
||||
// Verify the original content was NOT modified
|
||||
var final = JSON.parse((await readFile(testOutput, "utf-8")).trim());
|
||||
expect(final.preserved).toBe(true);
|
||||
} finally {
|
||||
try { await unlink(testOutput); } catch(e2) {}
|
||||
}
|
||||
});
|
||||
|
||||
it("two competing publication attempts cannot both claim the same destination", async function() {
|
||||
var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_competing.json");
|
||||
var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
|
||||
|
||||
try {
|
||||
// First publication — succeeds
|
||||
var result1 = await runWrapper([
|
||||
"--condition", "D",
|
||||
"--file", fixturePath,
|
||||
"--output", testOutput
|
||||
]);
|
||||
expect(result1.exitCode).toBe(0);
|
||||
expect(result1.error).toBe(false);
|
||||
|
||||
// Verify winning evidence bytes
|
||||
var rawBytes1 = await readFile(testOutput, "utf-8");
|
||||
var evidence1 = JSON.parse(rawBytes1.trim());
|
||||
expect(evidence1.success).toBe(true);
|
||||
expect(evidence1.statusCode).toBe(200);
|
||||
expect(evidence1.condition).toBe("D");
|
||||
|
||||
// Simulate a competing claim: write different content to the same destination
|
||||
var competingContent = JSON.stringify({ overwrittenBy: "competing-claim" });
|
||||
await writeFile(testOutput, competingContent, "utf-8");
|
||||
|
||||
// Second publication to the same path — must fail (EEXIST)
|
||||
var result2 = await runWrapper([
|
||||
"--condition", "D",
|
||||
"--file", fixturePath,
|
||||
"--output", testOutput
|
||||
]);
|
||||
expect(result2.exitCode).not.toBe(0);
|
||||
|
||||
// Verify winning evidence bytes remain unchanged:
|
||||
// file still has the competing claim content (second wrapper did NOT overwrite it)
|
||||
var rawBytes2 = await readFile(testOutput, "utf-8");
|
||||
expect(rawBytes2).toBe(competingContent);
|
||||
|
||||
// Also verify first publication bytes are intact — they differ from current file,
|
||||
// confirming the second wrapper could not reach them
|
||||
expect(rawBytes2).not.toBe(rawBytes1);
|
||||
} finally {
|
||||
try { await unlink(testOutput); } catch(e2) {}
|
||||
}
|
||||
});
|
||||
|
||||
it("linkSync hardlink publication rejects EEXIST when a second process targets an already-published destination", async function() {
|
||||
var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_link_a.json");
|
||||
var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
|
||||
|
||||
try {
|
||||
// First wrapper run — publishes successfully (linkSync creates hardlink at testOutput)
|
||||
var resultA = await runWrapper([
|
||||
"--condition", "D",
|
||||
"--file", fixturePath,
|
||||
"--output", testOutput
|
||||
]);
|
||||
expect(resultA.exitCode).toBe(0);
|
||||
|
||||
// Verify the first publication produced valid evidence
|
||||
var publishedA = JSON.parse((await readFile(testOutput, "utf-8")).trim());
|
||||
expect(publishedA.success).toBe(true);
|
||||
expect(publishedA.statusCode).toBe(200);
|
||||
expect(publishedA.condition).toBe("D");
|
||||
expect(typeof publishedA.freeformText).toBe("string");
|
||||
expect(publishedA.diagnostics.reconstructionOnly).toBe(true);
|
||||
|
||||
// Second wrapper targets the exact same destination path.
|
||||
// linkSync(tmpFile, testOutput) must throw EEXIST because testOutput already exists
|
||||
// as a hardlink to the first publication's data (same inode).
|
||||
var resultB = await runWrapper([
|
||||
"--condition", "D",
|
||||
"--file", fixturePath,
|
||||
"--output", testOutput
|
||||
]);
|
||||
expect(resultB.exitCode).not.toBe(0);
|
||||
|
||||
// Verify the first publication's evidence bytes remain exactly as published:
|
||||
// no-write guarantee confirmed — linkSync refuses to modify existing destination.
|
||||
var preserved = JSON.parse((await readFile(testOutput, "utf-8")).trim());
|
||||
expect(preserved.success).toBe(true);
|
||||
expect(preserved.statusCode).toBe(200);
|
||||
expect(preserved.condition).toBe("D");
|
||||
} finally {
|
||||
try { await unlink(testOutput); } catch(e) {}
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user