1352 lines
52 KiB
JavaScript
1352 lines
52 KiB
JavaScript
/**
|
|
* Deterministic apparatus verification for start-case-experiment-helper.cjs.
|
|
*
|
|
* Proves: standalone execution plumbing — env loading, input parsing,
|
|
* structured output, exit codes, no-retry behavior.
|
|
*
|
|
* Uses execFile subprocess invocation to test the actual CLI path.
|
|
* Mock mode via START_CASE_EXPERIMENT_HELPER_MOCK=1 replaces startCase
|
|
* with a deterministic inline double (zero live model calls).
|
|
*/
|
|
|
|
import { describe, expect, it, beforeAll, afterAll, vi } from "vitest";
|
|
import { execFile } from "child_process";
|
|
import { readFile, writeFile, unlink } from "fs/promises";
|
|
import { join, dirname } from "path";
|
|
import { fileURLToPath } from "url";
|
|
import { createRequire } from "module";
|
|
|
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
const rootDir = join(__dirname, "..", "..");
|
|
const helperPath = join(rootDir, "scripts", "start-case-experiment-helper.cjs");
|
|
const tsxPath = join(rootDir, "node_modules", ".bin", "tsx");
|
|
const require = createRequire(import.meta.url);
|
|
|
|
function runHelper(args = [], envOverrides = {}) {
|
|
return new Promise((resolve, reject) => {
|
|
execFile(
|
|
process.execPath,
|
|
[helperPath, ...args],
|
|
{
|
|
cwd: rootDir,
|
|
timeout: 15000,
|
|
env: {
|
|
...process.env,
|
|
// Always clear any real model config to avoid live calls
|
|
OLLAMA_BASE_URL: "https://mock.host.example.com:8080",
|
|
OLLAMA_MODEL: "mock-model-for-testing",
|
|
START_CASE_EXPERIMENT_HELPER_MOCK: "1",
|
|
...envOverrides,
|
|
},
|
|
},
|
|
(error, stdout, stderr) => {
|
|
resolve({
|
|
error: !!error,
|
|
exitCode: error ? error.code : 0,
|
|
stdout: stdout || "",
|
|
stderr: stderr || "",
|
|
});
|
|
}
|
|
);
|
|
});
|
|
}
|
|
|
|
function runHelperWithTsx(args = [], envOverrides = {}) {
|
|
return new Promise((resolve) => {
|
|
execFile(
|
|
tsxPath,
|
|
[helperPath, ...args],
|
|
{
|
|
cwd: rootDir,
|
|
timeout: 15000,
|
|
env: {
|
|
...process.env,
|
|
OLLAMA_BASE_URL: "https://mock.host.example.com:8080",
|
|
OLLAMA_MODEL: "mock-model-for-testing",
|
|
START_CASE_EXPERIMENT_HELPER_MOCK: "1",
|
|
...envOverrides,
|
|
},
|
|
},
|
|
(error, stdout, stderr) => {
|
|
resolve({
|
|
error: !!error,
|
|
exitCode: error ? error.code : 0,
|
|
stdout: stdout || "",
|
|
stderr: stderr || "",
|
|
});
|
|
},
|
|
);
|
|
});
|
|
}
|
|
|
|
function parseOutput(result) {
|
|
try {
|
|
return JSON.parse(result.stdout.trim());
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
describe("start-case-experiment-helper.cjs apparatus", () => {
|
|
let tmpFile = "";
|
|
let metadataFile = "";
|
|
|
|
beforeAll(async () => {
|
|
// Create a temporary scenario fixture for file-input tests
|
|
tmpFile = join(rootDir, "tests", "fixtures", "_helper_scenario_temp.json");
|
|
await writeFile(tmpFile, JSON.stringify({ scenario: "Scenario from file" }), "utf-8");
|
|
metadataFile = join(rootDir, "tests", "fixtures", "_helper_semantic_scenario_temp.json");
|
|
await writeFile(
|
|
metadataFile,
|
|
JSON.stringify({
|
|
experimentId: "semantic-open-question-001",
|
|
scenarioId: "manufacturing-expansion-001",
|
|
scenario: "Scenario with semantic inspection metadata",
|
|
promptVersion: "v0.5",
|
|
}),
|
|
"utf-8",
|
|
);
|
|
});
|
|
|
|
afterAll(async () => {
|
|
// Clean up temp fixture
|
|
try { await unlink(tmpFile); } catch {}
|
|
try { await unlink(metadataFile); } catch {}
|
|
});
|
|
|
|
// ── A — Positional scenario input ───────────────────────────────
|
|
|
|
describe("A — positional scenario input", () => {
|
|
it("passes 'Scenario text' to startCase exactly once", async () => {
|
|
const result = await runHelper(["Scenario text"]);
|
|
expect(result.error).toBe(false);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toBeNull();
|
|
expect(typeof parsed.success).toBe("boolean");
|
|
});
|
|
});
|
|
|
|
// ── B — File input ──────────────────────────────────────────────
|
|
|
|
describe("B — file input", () => {
|
|
it("reads scenario from JSON fixture", async () => {
|
|
const result = await runHelper(["--file", tmpFile]);
|
|
expect(result.error).toBe(false);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toBeNull();
|
|
expect(typeof parsed.success).toBe("boolean");
|
|
});
|
|
|
|
it("--file with nonexistent path fails cleanly", async () => {
|
|
const result = await runHelper(["--file", "/nonexistent/path.json"]);
|
|
// Should fail before startCase (file not found)
|
|
expect(result.exitCode).toBe(1);
|
|
const parsed = parseOutput(result);
|
|
if (parsed) {
|
|
expect(parsed.success).toBe(false);
|
|
expect(typeof parsed.error).toBe("string");
|
|
} else {
|
|
// Non-JSON failure output is acceptable for file-not-found edge case
|
|
expect(result.stderr.length + result.stdout.length).toBeGreaterThan(0);
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("B.1 — reconstruction-only CLI option", () => {
|
|
it("passes reconstructionOnly through the startCase seam without selecting OpenAI", async () => {
|
|
const result = await runHelper(["--reconstruction-only", "Scenario text"]);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed.diagnostics).toMatchObject({
|
|
modelName: "inline-test-double",
|
|
reconstructionOnly: true,
|
|
});
|
|
});
|
|
|
|
it("composes reconstruction-only with file input", async () => {
|
|
const result = await runHelper(["--reconstruction-only", "--file", tmpFile]);
|
|
expect(result.exitCode).toBe(0);
|
|
expect(parseOutput(result).diagnostics.reconstructionOnly).toBe(true);
|
|
});
|
|
|
|
it("preserves experiment metadata and exposes semantic evidence", async () => {
|
|
const result = await runHelper(["--reconstruction-only", "--file", metadataFile]);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
|
|
expect(parsed.experimentId).toBe("semantic-open-question-001");
|
|
expect(parsed.scenarioId).toBe("manufacturing-expansion-001");
|
|
expect(parsed.execution).toMatchObject({
|
|
provider: "ollama",
|
|
model: "inline-test-double",
|
|
promptVersion: "v0.5",
|
|
});
|
|
expect(parsed.reconstruction.importantUnknowns).toEqual([
|
|
{
|
|
id: "unknown-1",
|
|
description: "Canonical uncertainty from the deterministic test double",
|
|
confidence: "medium",
|
|
question: "What would resolve the deterministic uncertainty?",
|
|
},
|
|
]);
|
|
expect(parsed.graphUnknownNodes).toEqual([
|
|
{
|
|
id: "graph-unknown-1",
|
|
label: "Canonical uncertainty",
|
|
description: "Canonical uncertainty from the deterministic test double",
|
|
openQuestion: "What would resolve the deterministic uncertainty?",
|
|
status: "unknown",
|
|
},
|
|
]);
|
|
expect(parsed.diagnostics).toMatchObject({
|
|
validationStatus: "mock",
|
|
presentationValidationStatus: "valid",
|
|
presentationValidationIssues: [],
|
|
graphReferenceValidation: { valid: true, errors: [] },
|
|
compatibilityApplied: false,
|
|
compatibilityChanges: [],
|
|
compatibilityWarnings: [],
|
|
});
|
|
});
|
|
|
|
it("passes only production-supported request fields and calls startCase once", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
const importantUnknowns = [
|
|
{
|
|
id: "unknown-a",
|
|
description: "Canonical uncertainty A",
|
|
confidence: "high",
|
|
question: "What resolves uncertainty A?",
|
|
},
|
|
];
|
|
const startCase = vi.fn(async () => ({
|
|
success: true,
|
|
reconstruction: { importantUnknowns },
|
|
situationGraph: {
|
|
nodes: [
|
|
{
|
|
id: "unknown-node-a",
|
|
label: "Canonical uncertainty A",
|
|
description: "Canonical uncertainty A",
|
|
openQuestion: "What resolves uncertainty A?",
|
|
status: "unknown",
|
|
kind: "unknown",
|
|
},
|
|
{
|
|
id: "observation-node-a",
|
|
label: "Known observation",
|
|
description: "Known observation",
|
|
status: "supported",
|
|
kind: "observation",
|
|
},
|
|
],
|
|
edges: [],
|
|
},
|
|
selectedQuestion: null,
|
|
summary: "test",
|
|
diagnostics: {
|
|
modelName: "configured-model",
|
|
promptVersion: "v0.5",
|
|
validationStatus: "valid",
|
|
presentationValidationStatus: "partial",
|
|
presentationValidationIssues: [{ category: "test-diagnostic" }],
|
|
graphReferenceValidation: { valid: true, errors: [] },
|
|
compatibilityApplied: true,
|
|
compatibilityChanges: ["normalised"],
|
|
compatibilityWarnings: ["warning"],
|
|
},
|
|
}));
|
|
|
|
const result = await runStartCaseExperiment(
|
|
{
|
|
experimentId: "experiment-a",
|
|
scenarioId: "scenario-a",
|
|
scenario: "Scenario A",
|
|
promptVersion: "v0.5",
|
|
ignoredMetadata: "must not cross the production boundary",
|
|
},
|
|
null,
|
|
{ startCase, reconstructionOnly: true },
|
|
);
|
|
|
|
expect(startCase).toHaveBeenCalledTimes(1);
|
|
expect(startCase).toHaveBeenCalledWith(
|
|
{ scenario: "Scenario A", promptVersion: "v0.5" },
|
|
expect.objectContaining({ reconstructionOnly: true }),
|
|
);
|
|
expect(result.experimentId).toBe("experiment-a");
|
|
expect(result.scenarioId).toBe("scenario-a");
|
|
expect(result.reconstruction.importantUnknowns).toBe(importantUnknowns);
|
|
expect(result.graphUnknownNodes).toEqual([
|
|
{
|
|
id: "unknown-node-a",
|
|
label: "Canonical uncertainty A",
|
|
description: "Canonical uncertainty A",
|
|
openQuestion: "What resolves uncertainty A?",
|
|
status: "unknown",
|
|
},
|
|
]);
|
|
expect(result.execution).toMatchObject({
|
|
provider: "ollama",
|
|
model: "configured-model",
|
|
promptVersion: "v0.5",
|
|
});
|
|
expect(result.diagnostics).toEqual({
|
|
modelName: "configured-model",
|
|
promptVersion: "v0.5",
|
|
validationStatus: "valid",
|
|
presentationValidationStatus: "partial",
|
|
presentationValidationIssues: [{ category: "test-diagnostic" }],
|
|
graphReferenceValidation: { valid: true, errors: [] },
|
|
compatibilityApplied: true,
|
|
compatibilityChanges: ["normalised"],
|
|
compatibilityWarnings: ["warning"],
|
|
});
|
|
});
|
|
});
|
|
|
|
// ── C — Environment loading ─────────────────────────────────────
|
|
|
|
describe("C — environment loading", () => {
|
|
it(".env.local values load without dotenv dependency", async () => {
|
|
const result = await runHelper(["Env load test"]);
|
|
expect(result.error).toBe(false);
|
|
expect(result.exitCode).toBe(0);
|
|
});
|
|
|
|
it("missing .env.local does not crash — proceeds with set env", async () => {
|
|
const result = await runHelper(["No dotenv needed"]);
|
|
expect(result.exitCode).toBe(0);
|
|
});
|
|
});
|
|
|
|
// ── D — Success output ──────────────────────────────────────────
|
|
|
|
describe("D — success output", () => {
|
|
it("selects the existing OpenAI provider through the helper injection seam", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
const previousProvider = process.env.START_CASE_EXPERIMENT_PROVIDER;
|
|
const previousKey = process.env.OPENAI_API_KEY;
|
|
process.env.START_CASE_EXPERIMENT_PROVIDER = "openai";
|
|
process.env.OPENAI_API_KEY = "test-key";
|
|
|
|
try {
|
|
const result = await runStartCaseExperiment(
|
|
{ scenario: "OpenAI provider selection scenario" },
|
|
null,
|
|
{
|
|
startCase: async (_input, dependencies) => {
|
|
expect(typeof dependencies.reconstructionProvider?.generateReconstruction).toBe("function");
|
|
expect(dependencies.reconstructionModelName).toBe("gpt-5.6-terra");
|
|
expect(dependencies.reconstructionOnly).toBe(true);
|
|
return {
|
|
success: true,
|
|
situationGraph: { nodes: [], edges: [] },
|
|
assessment: null,
|
|
selectedQuestion: null,
|
|
summary: "test",
|
|
};
|
|
},
|
|
},
|
|
);
|
|
|
|
expect(result.success).toBe(true);
|
|
} finally {
|
|
if (previousProvider === undefined) delete process.env.START_CASE_EXPERIMENT_PROVIDER;
|
|
else process.env.START_CASE_EXPERIMENT_PROVIDER = previousProvider;
|
|
if (previousKey === undefined) delete process.env.OPENAI_API_KEY;
|
|
else process.env.OPENAI_API_KEY = previousKey;
|
|
}
|
|
});
|
|
|
|
it("fails before startCase when OpenAI is selected without a key", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
const previousProvider = process.env.START_CASE_EXPERIMENT_PROVIDER;
|
|
const previousKey = process.env.OPENAI_API_KEY;
|
|
process.env.START_CASE_EXPERIMENT_PROVIDER = "openai";
|
|
delete process.env.OPENAI_API_KEY;
|
|
const startCase = vi.fn();
|
|
|
|
try {
|
|
await expect(
|
|
runStartCaseExperiment({ scenario: "Missing key scenario" }, null, { startCase }),
|
|
).rejects.toThrow("START_CASE_EXPERIMENT_PROVIDER=openai requires OPENAI_API_KEY");
|
|
expect(startCase).not.toHaveBeenCalled();
|
|
} finally {
|
|
if (previousProvider === undefined) delete process.env.START_CASE_EXPERIMENT_PROVIDER;
|
|
else process.env.START_CASE_EXPERIMENT_PROVIDER = previousProvider;
|
|
if (previousKey === undefined) delete process.env.OPENAI_API_KEY;
|
|
else process.env.OPENAI_API_KEY = previousKey;
|
|
}
|
|
});
|
|
|
|
it("forwards an explicit reconstruction provider through the startCase path", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
const reconstructionProvider = { generateReconstruction() {} };
|
|
const startCase = async (input, dependencies) => {
|
|
expect(input).toEqual({ scenario: "Provider seam scenario" });
|
|
expect(dependencies).toMatchObject({
|
|
reconstructionProvider,
|
|
reconstructionModelName: "gpt-5.6-terra",
|
|
});
|
|
return {
|
|
success: true,
|
|
situationGraph: { nodes: [], edges: [] },
|
|
assessment: null,
|
|
selectedQuestion: null,
|
|
summary: "test",
|
|
};
|
|
};
|
|
|
|
const result = await runStartCaseExperiment(
|
|
{ scenario: "Provider seam scenario" },
|
|
null,
|
|
{ reconstructionProvider, reconstructionModelName: "gpt-5.6-terra", startCase },
|
|
);
|
|
|
|
expect(result.success).toBe(true);
|
|
expect(result.summary).toBe("test");
|
|
});
|
|
|
|
it("exit code is 0", async () => {
|
|
const result = await runHelper(["Success test scenario"]);
|
|
expect(result.exitCode).toBe(0);
|
|
});
|
|
|
|
it("stdout is valid JSON", async () => {
|
|
const result = await runHelper(["Valid JSON output"]);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toBeNull();
|
|
expect(typeof parsed).toBe("object");
|
|
});
|
|
|
|
it("success fields are present", async () => {
|
|
const result = await runHelper(["Fields presence test"]);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).toHaveProperty("success");
|
|
expect(parsed.success).toBe(true);
|
|
expect(typeof parsed.situationGraphNodeCount).toBe("number");
|
|
expect(typeof parsed.endToEndElapsedMs).toBe("number");
|
|
});
|
|
|
|
it("endToEndElapsedMs is a non-negative number", async () => {
|
|
const result = await runHelper(["Timing test"]);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed.endToEndElapsedMs).toBeGreaterThanOrEqual(0);
|
|
expect(typeof parsed.endToEndElapsedMs).toBe("number");
|
|
});
|
|
|
|
it("error is null on success", async () => {
|
|
const result = await runHelper(["Error null test"]);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed.error).toBeNull();
|
|
});
|
|
|
|
it("preserves the existing non-reconstruction-only output contract", async () => {
|
|
const result = await runHelper(["Compatibility scenario"]);
|
|
const parsed = parseOutput(result);
|
|
|
|
expect(parsed).not.toHaveProperty("experimentId");
|
|
expect(parsed).not.toHaveProperty("scenarioId");
|
|
expect(parsed).not.toHaveProperty("execution");
|
|
expect(parsed).not.toHaveProperty("graphUnknownNodes");
|
|
expect(parsed).toHaveProperty("situationGraphNodeCount");
|
|
expect(parsed).toHaveProperty("situationGraphEdgeCount");
|
|
expect(parsed).toHaveProperty("endToEndElapsedMs");
|
|
});
|
|
});
|
|
|
|
// ── E — Failure output (startCase throws) ───────────────────────
|
|
|
|
describe("E — failure output (mock-fail scenario)", () => {
|
|
it("produces failure result with non-zero exit code via mock", async () => {
|
|
const result = await runHelper(["Failure test"], {
|
|
START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1",
|
|
});
|
|
expect(result.exitCode).not.toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toBeNull();
|
|
expect(parsed.success).toBe(false);
|
|
expect(typeof parsed.error).toBe("string");
|
|
});
|
|
|
|
it("failure output is valid JSON", async () => {
|
|
const result = await runHelper(["Failure test"], {
|
|
START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1",
|
|
});
|
|
if (result.stdout.trim().length > 0) {
|
|
const parsed = JSON.parse(result.stdout.trim());
|
|
expect(parsed).toHaveProperty("success", false);
|
|
}
|
|
});
|
|
});
|
|
|
|
// ── F — No retry ────────────────────────────────────────────────
|
|
|
|
describe("F — no retry", () => {
|
|
it("startCase is called exactly once on success", async () => {
|
|
const startTime = Date.now();
|
|
const result = await runHelper(["No retry test"]);
|
|
expect(result.error).toBe(false);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
// Single JSON output confirms no retries (would produce multiple results)
|
|
const lineCount = result.stdout.trim().split("\n").filter(l => l.trim()).length;
|
|
expect(lineCount).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("startCase is called exactly once on failure", async () => {
|
|
const result = await runHelper(["Failure retry test"], {
|
|
START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1",
|
|
});
|
|
// Should exit non-zero but only produce one JSON output (no retry)
|
|
expect(result.exitCode).not.toBe(0);
|
|
const parsed = parseOutput(result);
|
|
if (parsed) {
|
|
expect(parsed.success).toBe(false);
|
|
// Only one output line — proves no retry loop
|
|
const lineCount = result.stdout.trim().split("\n").filter(l => l.trim()).length;
|
|
expect(lineCount).toBeGreaterThan(0);
|
|
}
|
|
});
|
|
});
|
|
|
|
// ── G — Malformed input ─────────────────────────────────────────
|
|
|
|
describe("G — malformed input", () => {
|
|
it("malformed --file content fails before startCase", async () => {
|
|
const malformedFile = join(rootDir, "tests", "fixtures", "_helper_malformed.json");
|
|
await writeFile(malformedFile, "{ not valid json }", "utf-8");
|
|
try {
|
|
const result = await runHelper(["--file", malformedFile]);
|
|
expect(result.exitCode).not.toBe(0);
|
|
} finally {
|
|
await unlink(malformedFile);
|
|
}
|
|
});
|
|
|
|
it("--file without path argument fails before startCase", async () => {
|
|
const result = await runHelper(["--file"]);
|
|
expect(result.exitCode).not.toBe(0);
|
|
const parsed = parseOutput(result);
|
|
if (parsed) {
|
|
expect(parsed.success).toBe(false);
|
|
}
|
|
});
|
|
});
|
|
|
|
// ── Output structure integrity ──────────────────────────────────
|
|
|
|
describe("output structure integrity", () => {
|
|
it("stdout is not corrupted by diagnostic output", async () => {
|
|
const result = await runHelper(["Clean stdout test"]);
|
|
const trimmed = result.stdout.trim();
|
|
const parsed = JSON.parse(trimmed);
|
|
expect(parsed).toHaveProperty("success");
|
|
// stderr should be empty (no diagnostic leakage to stdout)
|
|
expect(result.stderr.length).toBe(0);
|
|
});
|
|
|
|
it("structured failure output is valid JSON", async () => {
|
|
const result = await runHelper(["Structured fail"], {
|
|
START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1",
|
|
});
|
|
expect(result.exitCode).not.toBe(0);
|
|
if (result.stdout.trim().length > 0) {
|
|
const parsed = JSON.parse(result.stdout.trim());
|
|
expect(parsed.success).toBe(false);
|
|
expect(typeof parsed.error).toBe("string");
|
|
}
|
|
});
|
|
|
|
it("contains no investigation-persistence invocation", async () => {
|
|
const source = await readFile(helperPath, "utf-8");
|
|
expect(source).not.toMatch(/investigation-storage|server-investigation-persistence|saveInvestigation|\/api\/investigations/);
|
|
});
|
|
});
|
|
|
|
// ── H — Import-only mode (production seam verification) ────────
|
|
|
|
describe("H — import-only mode", () => {
|
|
it("loads real startCase through tsx with aliases resolved and no provider call", async () => {
|
|
const result = await runHelperWithTsx([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
expect(result.exitCode).toBe(0);
|
|
expect(parseOutput(result)).toEqual({
|
|
success: true,
|
|
mode: "import-only",
|
|
startCaseResolved: true,
|
|
});
|
|
});
|
|
|
|
it("uses plain node subprocess with IMPORT_ONLY=1", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
expect(result.exitCode).not.toBe(0);
|
|
// exit non-zero because @/ alias blocks real orchestrator import (apparatus evidence)
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toBeNull();
|
|
expect(typeof parsed.success).toBe("boolean");
|
|
});
|
|
|
|
it("stdout contains exactly one valid JSON result with mode=import-only", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
const trimmed = result.stdout.trim();
|
|
expect(trimmed.split("\n").filter(l => l.trim()).length).toBe(1);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toBeNull();
|
|
expect(parsed.mode).toBe("import-only");
|
|
});
|
|
|
|
it("startCaseResolved is documented (false when @/ alias blocks import)", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).toHaveProperty("startCaseResolved");
|
|
// startCaseResolved reflects whether the real production import succeeded
|
|
expect(typeof parsed.startCaseResolved).toBe("boolean");
|
|
});
|
|
|
|
it("documented failureReason is a string (module resolution evidence)", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).toHaveProperty("failureReason");
|
|
expect(typeof parsed.failureReason).toBe("string");
|
|
expect(parsed.failureReason.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("no semantic result fields present (no situationGraph, no error from startCase)", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toHaveProperty("situationGraphNodeCount");
|
|
expect(parsed).not.toHaveProperty("situationGraphEdgeCount");
|
|
expect(parsed).not.toHaveProperty("assessmentPhase");
|
|
expect(parsed).not.toHaveProperty("endToEndElapsedMs");
|
|
});
|
|
|
|
it("no retry — single JSON output only", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
const lineCount = result.stdout.trim().split("\n").filter(l => l.trim()).length;
|
|
expect(lineCount).toBe(1);
|
|
});
|
|
|
|
it("no network/provider invocation — import-only mode structurally stops before any provider use", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
// The process exits with the documented import failure reason.
|
|
// It never reaches OLLAMA_BASE_URL check or startCase invocation.
|
|
const parsed = parseOutput(result);
|
|
expect(parsed.mode).toBe("import-only");
|
|
expect(typeof parsed.failureReason).toBe("string");
|
|
});
|
|
});
|
|
|
|
// ── I — Condition B schema variant apparatus ────────────────────
|
|
|
|
describe("I — Condition B schema variant apparatus", () => {
|
|
it("--condition B passes generationSchema through the startCase seam", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
let receivedDependencies;
|
|
|
|
const result = await runStartCaseExperiment(
|
|
{ scenario: "Condition B test scenario" },
|
|
null,
|
|
{
|
|
condition: "B",
|
|
startCase: async (_input, dependencies) => {
|
|
receivedDependencies = dependencies;
|
|
return {
|
|
success: true,
|
|
situationGraph: { nodes: [], edges: [] },
|
|
assessment: null,
|
|
selectedQuestion: null,
|
|
summary: "test",
|
|
diagnostics: { validationStatus: "valid" },
|
|
};
|
|
},
|
|
},
|
|
);
|
|
|
|
expect(result.success).toBe(true);
|
|
expect(receivedDependencies.generationSchema).toBeDefined();
|
|
// Condition B schema: importantUnknowns items should NOT have question property
|
|
const iuSchema = receivedDependencies.generationSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items;
|
|
if (iuSchema && iuSchema.properties) {
|
|
expect(iuSchema.properties).not.toHaveProperty("question");
|
|
}
|
|
});
|
|
|
|
it("--condition A uses production schema with question required", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
let receivedDependencies;
|
|
|
|
const result = await runStartCaseExperiment(
|
|
{ scenario: "Condition A test scenario" },
|
|
null,
|
|
{
|
|
condition: "A",
|
|
startCase: async (_input, dependencies) => {
|
|
receivedDependencies = dependencies;
|
|
return {
|
|
success: true,
|
|
situationGraph: { nodes: [], edges: [] },
|
|
assessment: null,
|
|
selectedQuestion: null,
|
|
summary: "test",
|
|
diagnostics: { validationStatus: "valid" },
|
|
};
|
|
},
|
|
},
|
|
);
|
|
|
|
expect(result.success).toBe(true);
|
|
});
|
|
|
|
it("--condition B with mock produces valid structured output", async () => {
|
|
const result = await runHelper(["--condition", "B", "Mock scenario"], {});
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed.success).toBe(true);
|
|
});
|
|
|
|
it("Condition B schema JSON has importantUnknowns WITHOUT question field", async () => {
|
|
const { conditionBJsonSchema, conditionAJsonSchema } = require(
|
|
join(rootDir, "lib", "reconstruction", "schema-variant.js"),
|
|
);
|
|
|
|
const bIuProps = conditionBJsonSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items?.properties;
|
|
const aIuProps = conditionAJsonSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items?.properties;
|
|
|
|
expect(bIuProps).toHaveProperty("id");
|
|
expect(bIuProps).toHaveProperty("description");
|
|
expect(bIuProps).toHaveProperty("confidence");
|
|
expect(bIuProps).not.toHaveProperty("question");
|
|
|
|
// Verify Condition A (production) still HAS question
|
|
expect(aIuProps).toHaveProperty("id");
|
|
expect(aIuProps).toHaveProperty("description");
|
|
expect(aIuProps).toHaveProperty("confidence");
|
|
expect(aIuProps).toHaveProperty("question");
|
|
});
|
|
|
|
it("Condition B prompt file exists and contains no question-generation language", async () => {
|
|
const fs = require("fs");
|
|
const altPromptPath = join(rootDir, "prompts", "reconstruct-v0.5-b-noreq.md");
|
|
|
|
expect(fs.existsSync(altPromptPath)).toBe(true);
|
|
const content = fs.readFileSync(altPromptPath, "utf8");
|
|
|
|
// The B prompt should NOT instruct model to generate questions for importantUnknowns
|
|
expect(content).not.toMatch(/express that same uncertainty in \`question\`/i);
|
|
expect(content).not.toMatch(/natural direct question/i);
|
|
|
|
// But Condition A prompt does contain this language
|
|
const prodPromptPath = join(rootDir, "prompts", "reconstruct-v0.5.md");
|
|
const prodContent = fs.readFileSync(prodPromptPath, "utf8");
|
|
expect(prodContent).toMatch(/express that same uncertainty in \`question\`/i);
|
|
});
|
|
|
|
it("Condition B prompt has identical line count to Condition A prompt", async () => {
|
|
const fs = require("fs");
|
|
const altLines = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-b-noreq.md"), "utf8").split("\n").length;
|
|
const prodLines = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5.md"), "utf8").split("\n").length;
|
|
expect(altLines).toBe(prodLines);
|
|
});
|
|
|
|
it("Condition B prompt differs from Condition A only in Rule 7a and output format example", async () => {
|
|
const fs = require("fs");
|
|
const altContent = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-b-noreq.md"), "utf8");
|
|
const prodContent = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5.md"), "utf8");
|
|
|
|
expect(altContent).not.toBe(prodContent);
|
|
expect(altContent).toMatch(/express its canonical uncertainty meaning as a description in \`description\` with strict semantic fidelity/i);
|
|
});
|
|
|
|
it("Condition B schema validates importantUnknowns WITHOUT question field required", async () => {
|
|
const { conditionBJsonSchema } = require(
|
|
join(rootDir, "lib", "reconstruction", "schema-variant.js"),
|
|
);
|
|
|
|
const bIuItems = conditionBJsonSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items;
|
|
expect(bIuItems).toBeDefined();
|
|
if (bIuItems && bIuItems.required) {
|
|
expect(bIuItems.required).not.toContain("question");
|
|
}
|
|
});
|
|
});
|
|
|
|
// ── J — Condition D treatment selector apparatus ────────────────
|
|
|
|
describe("J — Condition D prompt selection", () => {
|
|
it("default Condition D (no --treatment) selects original deconstruct prompt", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
let receivedScenario;
|
|
|
|
await runStartCaseExperiment(
|
|
{ scenario: "Condition D default test" },
|
|
null,
|
|
{
|
|
condition: "D",
|
|
startCase: async (input) => {
|
|
receivedScenario = input.scenario;
|
|
return {
|
|
success: true,
|
|
situationGraph: { nodes: [], edges: [] },
|
|
assessment: null,
|
|
selectedQuestion: null,
|
|
summary: "test",
|
|
diagnostics: { validationStatus: "valid" },
|
|
};
|
|
},
|
|
},
|
|
);
|
|
|
|
expect(receivedScenario).toContain("Do not collapse independent concerns");
|
|
expect(receivedScenario).toContain("Do not create artificial uncertainties to increase their number.");
|
|
});
|
|
|
|
it("--condition D --treatment no21 selects the treatment prompt (rule 21 removed)", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
let receivedScenario;
|
|
|
|
await runStartCaseExperiment(
|
|
{ scenario: "Condition D treatment test" },
|
|
null,
|
|
{
|
|
condition: "D",
|
|
treatment: "no21",
|
|
startCase: async (input) => {
|
|
receivedScenario = input.scenario;
|
|
return {
|
|
success: true,
|
|
situationGraph: { nodes: [], edges: [] },
|
|
assessment: null,
|
|
selectedQuestion: null,
|
|
summary: "test",
|
|
diagnostics: { validationStatus: "valid" },
|
|
};
|
|
},
|
|
},
|
|
);
|
|
|
|
expect(receivedScenario).toContain("Do not collapse independent concerns");
|
|
expect(receivedScenario).not.toContain("Do not create artificial uncertainties to increase their number.");
|
|
});
|
|
|
|
it("--condition D produces freeformText output field, not reconstruction", async () => {
|
|
const result = await runHelper(["--condition", "D", "Freeform test"]);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).toHaveProperty("freeformText");
|
|
expect(parsed.conditionType).toBe("deconstruct");
|
|
});
|
|
|
|
it("--condition D sets condition to D in output", async () => {
|
|
const result = await runHelper(["--condition", "D", "Condition D flag test"]);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed.condition).toBe("D");
|
|
});
|
|
});
|
|
|
|
describe("J.1 — Treatment validation", () => {
|
|
it("rejects unknown treatment value via programmatic API", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
|
|
await expect(
|
|
runStartCaseExperiment(
|
|
{ scenario: "Invalid treatment test" },
|
|
null,
|
|
{
|
|
condition: "D",
|
|
treatment: "invalid-treatment",
|
|
startCase: async () => ({ success: true }),
|
|
},
|
|
),
|
|
).rejects.toThrow("Invalid treatment: invalid-treatment. Must be one of: no21");
|
|
});
|
|
|
|
it("rejects treatment with non-D condition via programmatic API", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
|
|
await expect(
|
|
runStartCaseExperiment(
|
|
{ scenario: "Cross-condition test" },
|
|
null,
|
|
{
|
|
condition: "B",
|
|
treatment: "no21",
|
|
startCase: async () => ({ success: true }),
|
|
},
|
|
),
|
|
).rejects.toThrow('Treatment "no21" is only valid with --condition D or E, not --condition B');
|
|
});
|
|
|
|
it("rejects unknown condition value via CLI", async () => {
|
|
const result = await runHelper(["--condition", "Z", "Invalid condition test"]);
|
|
expect(result.exitCode).not.toBe(0);
|
|
const parsed = parseOutput(result);
|
|
if (parsed) {
|
|
expect(parsed.success).toBe(false);
|
|
expect(typeof parsed.error).toBe("string");
|
|
}
|
|
});
|
|
|
|
it("rejects treatment with non-D condition via CLI", async () => {
|
|
const result = await runHelper(["--condition", "B", "--treatment", "no21", "Cross-condition CLI test"]);
|
|
expect(result.exitCode).not.toBe(0);
|
|
const parsed = parseOutput(result);
|
|
if (parsed) {
|
|
expect(parsed.success).toBe(false);
|
|
expect(typeof parsed.error).toBe("string");
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("J.2 — Prompt isolation verification", () => {
|
|
it("original and treatment prompts differ only by removal of rule 21", async () => {
|
|
const fs = require("fs");
|
|
const path = require("path");
|
|
const originalPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct.md"), "utf8");
|
|
const treatmentPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct-no21.md"), "utf8");
|
|
|
|
expect(originalPrompt).not.toBe(treatmentPrompt);
|
|
|
|
// The only difference should be the removal of rule 21 line and its blank line
|
|
const originalLines = originalPrompt.split("\n");
|
|
const treatmentLines = treatmentPrompt.split("\n");
|
|
|
|
// Find where they diverge — rule 21 is on line 21
|
|
expect(originalLines[20]).toBe("Do not create artificial uncertainties to increase their number.");
|
|
|
|
// After removal, everything else should shift by exactly one line
|
|
for (let i = 21; i < treatmentLines.length; i++) {
|
|
expect(originalLines[i + 1]).toBe(treatmentLines[i]);
|
|
}
|
|
});
|
|
|
|
it("both prompts share identical preamble through rule 20", async () => {
|
|
const fs = require("fs");
|
|
const originalPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct.md"), "utf8");
|
|
const treatmentPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct-no21.md"), "utf8");
|
|
|
|
// First 20 lines should be identical (before rule 21)
|
|
const originalPreamble = originalPrompt.split("\n").slice(0, 20).join("\n");
|
|
const treatmentPreamble = treatmentPrompt.split("\n").slice(0, 20).join("\n");
|
|
expect(originalPreamble).toBe(treatmentPreamble);
|
|
});
|
|
|
|
it("original prompt still contains rule 21 verbatim", async () => {
|
|
const fs = require("fs");
|
|
const originalPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct.md"), "utf8");
|
|
expect(originalPrompt).toContain("Do not create artificial uncertainties to increase their number.");
|
|
});
|
|
|
|
it("treatment prompt does NOT contain rule 21", async () => {
|
|
const fs = require("fs");
|
|
const treatmentPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct-no21.md"), "utf8");
|
|
expect(treatmentPrompt).not.toContain("Do not create artificial uncertainties to increase their number.");
|
|
});
|
|
|
|
it("Condition D output cannot overwrite original Condition D evidence — distinct freeformText field", async () => {
|
|
const resultTreatment = await runHelper(["--condition", "D", "--treatment", "no21", "Treatment output test"]);
|
|
const resultOriginal = await runHelper(["--condition", "D", "Original Condition D test"]);
|
|
|
|
expect(resultTreatment.exitCode).toBe(0);
|
|
expect(resultOriginal.exitCode).toBe(0);
|
|
|
|
const treatmentOutput = parseOutput(resultTreatment);
|
|
const originalOutput = parseOutput(resultOriginal);
|
|
|
|
// Both should have freeformText (not reconstruction) for Condition D
|
|
expect(treatmentOutput).toHaveProperty("freeformText");
|
|
expect(originalOutput).toHaveProperty("freeformText");
|
|
expect(treatmentOutput).not.toHaveProperty("reconstruction");
|
|
expect(originalOutput).not.toHaveProperty("reconstruction");
|
|
|
|
// They use the same output structure (freeformText) but different content —
|
|
// the test double returns deterministic text that confirms prompt selection works
|
|
expect(typeof treatmentOutput.freeformText).toBe("string");
|
|
expect(typeof originalOutput.freeformText).toBe("string");
|
|
});
|
|
});
|
|
|
|
// ── K — Condition E investigability-first apparatus ────────
|
|
|
|
describe("K — Condition E investigability-first apparatus", () => {
|
|
it("--condition E selects the investigability-first prompt rather than deconstruct", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
let receivedScenario;
|
|
|
|
await runStartCaseExperiment(
|
|
{ scenario: "Condition E test" },
|
|
null,
|
|
{
|
|
condition: "E",
|
|
startCase: async (input) => {
|
|
receivedScenario = input.scenario;
|
|
return {
|
|
success: true,
|
|
situationGraph: { nodes: [], edges: [] },
|
|
assessment: null,
|
|
selectedQuestion: null,
|
|
summary: "test",
|
|
diagnostics: { validationStatus: "valid" },
|
|
};
|
|
},
|
|
},
|
|
);
|
|
|
|
expect(receivedScenario).toContain("You are a neutral analyst performing deconstruction of the user's situation");
|
|
expect(receivedScenario).toContain("Do not create artificial uncertainties to increase their number.");
|
|
});
|
|
|
|
it("--condition E --treatment e-investigability-first is accepted", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
|
|
await runStartCaseExperiment(
|
|
{ scenario: "Condition E treatment test" },
|
|
null,
|
|
{
|
|
condition: "E",
|
|
treatment: "e-investigability-first",
|
|
startCase: async (input) => {
|
|
return {
|
|
success: true,
|
|
situationGraph: { nodes: [], edges: [] },
|
|
assessment: null,
|
|
selectedQuestion: null,
|
|
summary: "test",
|
|
diagnostics: { validationStatus: "valid" },
|
|
};
|
|
},
|
|
},
|
|
);
|
|
|
|
// If we reach here without an error, the treatment was accepted
|
|
});
|
|
|
|
it("--condition E produces freeformText output field via CLI mock", async () => {
|
|
const result = await runHelper(["--condition", "E", "Freeform test"]);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).toHaveProperty("freeformText");
|
|
expect(typeof parsed.freeformText).toBe("string");
|
|
expect(parsed.conditionType).toBe("investigability-first");
|
|
});
|
|
|
|
it("--condition E sets condition to E in output", async () => {
|
|
const result = await runHelper(["--condition", "E", "Condition E flag test"]);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed.condition).toBe("E");
|
|
});
|
|
|
|
it("--condition E does not return reconstruction field in output", async () => {
|
|
const result = await runHelper(["--condition", "E", "No reconstruction test"]);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toHaveProperty("reconstruction");
|
|
});
|
|
|
|
it("--condition E --reconstruction-only sets execution.condition to E", async () => {
|
|
const result = await runHelper(["--condition", "E", "--reconstruction-only", "Reconstruction only test"]);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed.execution).toBeDefined();
|
|
expect(parsed.execution.condition).toBe("E");
|
|
});
|
|
|
|
it("--condition E rejects unknown treatment via programmatic API", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
|
|
await expect(
|
|
runStartCaseExperiment(
|
|
{ scenario: "Invalid treatment test" },
|
|
null,
|
|
{
|
|
condition: "E",
|
|
treatment: "invalid-treatment",
|
|
startCase: async () => ({ success: true }),
|
|
},
|
|
),
|
|
).rejects.toThrow("Invalid treatment: invalid-treatment. Must be one of: e-investigability-first");
|
|
});
|
|
|
|
it("--condition E rejects treatment with non-E condition via programmatic API", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
|
|
await expect(
|
|
runStartCaseExperiment(
|
|
{ scenario: "Cross-condition test" },
|
|
null,
|
|
{
|
|
condition: "B",
|
|
treatment: "e-investigability-first",
|
|
startCase: async () => ({ success: true }),
|
|
},
|
|
),
|
|
).rejects.toThrow('Treatment "e-investigability-first" is only valid with --condition D or E, not --condition B');
|
|
});
|
|
|
|
it("--condition E exposes non-empty freeformText (mock)", async () => {
|
|
const result = await runHelper(["--condition", "E", "Freeform content test"]);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed.freeformText.length).toBeGreaterThan(0);
|
|
});
|
|
});
|
|
|
|
});
|
|
|
|
// ── Evidence persistence wrapper tests ─────────────────────────────
|
|
|
|
var wrapperPath = join(rootDir, "scripts", "evidence-persist.cjs");
|
|
|
|
function runWrapper(args, envOverrides) {
|
|
return new Promise(function(resolve) {
|
|
execFile(
|
|
process.execPath,
|
|
[wrapperPath].concat(args),
|
|
{
|
|
cwd: rootDir,
|
|
timeout: 30000,
|
|
env: Object.assign({}, process.env, {
|
|
START_CASE_EXPERIMENT_HELPER_MOCK: "1",
|
|
}, envOverrides || {}),
|
|
},
|
|
function(error, stdout, stderr) {
|
|
resolve({
|
|
error: !!error,
|
|
exitCode: error ? error.code : 0,
|
|
stdout: stdout || "",
|
|
stderr: stderr || "",
|
|
});
|
|
}
|
|
);
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Tests for the evidence-persist.cjs wrapper.
|
|
*
|
|
* Uses START_CASE_EXPERIMENT_HELPER_MOCK=1 for deterministic behavior.
|
|
*/
|
|
|
|
describe("evidence persistence", function() {
|
|
|
|
it("successful mock invocation publishes valid evidence with exact raw stdout bytes preserved", async function() {
|
|
var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_d.json");
|
|
var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
|
|
|
|
try {
|
|
var result = await runWrapper([
|
|
"--condition", "D",
|
|
"--file", fixturePath,
|
|
"--output", testOutput
|
|
]);
|
|
|
|
expect(result.exitCode).toBe(0);
|
|
expect(result.error).toBe(false);
|
|
|
|
// Verify the evidence file was published
|
|
var statSync = require("fs").statSync;
|
|
var exists = false;
|
|
try { statSync(testOutput); exists = true; } catch(e) {}
|
|
expect(exists).toBe(true);
|
|
|
|
// Verify the published file is valid JSON matching what helper outputs
|
|
var rawContent = await readFile(testOutput, "utf-8");
|
|
var published = JSON.parse(rawContent.trim());
|
|
expect(published.success).toBe(true);
|
|
expect(published.statusCode).toBe(200);
|
|
expect(published.condition).toBe("D");
|
|
expect(typeof published.freeformText).toBe("string");
|
|
expect(published.freeformText.length).toBeGreaterThan(0);
|
|
expect(published.diagnostics.reconstructionOnly).toBe(true);
|
|
|
|
// Verify raw bytes are preserved (no reformatting)
|
|
var reparseable = JSON.parse(rawContent.trim());
|
|
expect(reparseable.condition).toBe("D");
|
|
} finally {
|
|
try { await unlink(testOutput); } catch(e2) {}
|
|
}
|
|
});
|
|
|
|
it("forced helper failure does not publish evidence", async function() {
|
|
var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_fail.json");
|
|
var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
|
|
|
|
try {
|
|
var result = await runWrapper([
|
|
"--condition", "D",
|
|
"--file", fixturePath,
|
|
"--output", testOutput
|
|
], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1" });
|
|
|
|
expect(result.exitCode).not.toBe(0);
|
|
|
|
// Verify the final destination was NOT created
|
|
var statSync = require("fs").statSync;
|
|
var exists = false;
|
|
try { statSync(testOutput); exists = true; } catch(e) {}
|
|
expect(exists).toBe(false);
|
|
} finally {
|
|
try { await unlink(testOutput); } catch(e2) {}
|
|
}
|
|
});
|
|
|
|
it("malformed or incomplete output is rejected without publication", async function() {
|
|
var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_malformed.json");
|
|
var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
|
|
|
|
try {
|
|
var result = await runWrapper([
|
|
"--condition", "D",
|
|
"--file", fixturePath,
|
|
"--output", testOutput
|
|
], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1" });
|
|
|
|
expect(result.exitCode).not.toBe(0);
|
|
|
|
// Verify destination was NOT created (rejection without publication)
|
|
var statSync2 = require("fs").statSync;
|
|
var exists = false;
|
|
try { statSync2(testOutput); exists = true; } catch(e) {}
|
|
expect(exists).toBe(false);
|
|
} finally {
|
|
try { await unlink(testOutput); } catch(e2) {}
|
|
}
|
|
});
|
|
|
|
it("an existing destination is preserved unchanged", async function() {
|
|
var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_existing.json");
|
|
var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
|
|
|
|
try {
|
|
// Pre-create the destination with known content
|
|
await writeFile(testOutput, JSON.stringify({ preserved: true }), "utf-8");
|
|
var original = JSON.parse((await readFile(testOutput, "utf-8")).trim());
|
|
expect(original.preserved).toBe(true);
|
|
|
|
// Attempt to publish — should be rejected due to existing destination
|
|
var result = await runWrapper([
|
|
"--condition", "D",
|
|
"--file", fixturePath,
|
|
"--output", testOutput
|
|
]);
|
|
|
|
expect(result.exitCode).not.toBe(0);
|
|
|
|
// Verify the original content was NOT modified
|
|
var final = JSON.parse((await readFile(testOutput, "utf-8")).trim());
|
|
expect(final.preserved).toBe(true);
|
|
} finally {
|
|
try { await unlink(testOutput); } catch(e2) {}
|
|
}
|
|
});
|
|
|
|
it("two competing publication attempts cannot both claim the same destination", async function() {
|
|
var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_competing.json");
|
|
var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
|
|
|
|
try {
|
|
// First publication — succeeds
|
|
var result1 = await runWrapper([
|
|
"--condition", "D",
|
|
"--file", fixturePath,
|
|
"--output", testOutput
|
|
]);
|
|
expect(result1.exitCode).toBe(0);
|
|
expect(result1.error).toBe(false);
|
|
|
|
// Verify winning evidence bytes
|
|
var rawBytes1 = await readFile(testOutput, "utf-8");
|
|
var evidence1 = JSON.parse(rawBytes1.trim());
|
|
expect(evidence1.success).toBe(true);
|
|
expect(evidence1.statusCode).toBe(200);
|
|
expect(evidence1.condition).toBe("D");
|
|
|
|
// Simulate a competing claim: write different content to the same destination
|
|
var competingContent = JSON.stringify({ overwrittenBy: "competing-claim" });
|
|
await writeFile(testOutput, competingContent, "utf-8");
|
|
|
|
// Second publication to the same path — must fail (EEXIST)
|
|
var result2 = await runWrapper([
|
|
"--condition", "D",
|
|
"--file", fixturePath,
|
|
"--output", testOutput
|
|
]);
|
|
expect(result2.exitCode).not.toBe(0);
|
|
|
|
// Verify winning evidence bytes remain unchanged:
|
|
// file still has the competing claim content (second wrapper did NOT overwrite it)
|
|
var rawBytes2 = await readFile(testOutput, "utf-8");
|
|
expect(rawBytes2).toBe(competingContent);
|
|
|
|
// Also verify first publication bytes are intact — they differ from current file,
|
|
// confirming the second wrapper could not reach them
|
|
expect(rawBytes2).not.toBe(rawBytes1);
|
|
} finally {
|
|
try { await unlink(testOutput); } catch(e2) {}
|
|
}
|
|
});
|
|
|
|
it("linkSync hardlink publication rejects EEXIST when a second process targets an already-published destination", async function() {
|
|
var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_link_a.json");
|
|
var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json");
|
|
|
|
try {
|
|
// First wrapper run — publishes successfully (linkSync creates hardlink at testOutput)
|
|
var resultA = await runWrapper([
|
|
"--condition", "D",
|
|
"--file", fixturePath,
|
|
"--output", testOutput
|
|
]);
|
|
expect(resultA.exitCode).toBe(0);
|
|
|
|
// Verify the first publication produced valid evidence
|
|
var publishedA = JSON.parse((await readFile(testOutput, "utf-8")).trim());
|
|
expect(publishedA.success).toBe(true);
|
|
expect(publishedA.statusCode).toBe(200);
|
|
expect(publishedA.condition).toBe("D");
|
|
expect(typeof publishedA.freeformText).toBe("string");
|
|
expect(publishedA.diagnostics.reconstructionOnly).toBe(true);
|
|
|
|
// Second wrapper targets the exact same destination path.
|
|
// linkSync(tmpFile, testOutput) must throw EEXIST because testOutput already exists
|
|
// as a hardlink to the first publication's data (same inode).
|
|
var resultB = await runWrapper([
|
|
"--condition", "D",
|
|
"--file", fixturePath,
|
|
"--output", testOutput
|
|
]);
|
|
expect(resultB.exitCode).not.toBe(0);
|
|
|
|
// Verify the first publication's evidence bytes remain exactly as published:
|
|
// no-write guarantee confirmed — linkSync refuses to modify existing destination.
|
|
var preserved = JSON.parse((await readFile(testOutput, "utf-8")).trim());
|
|
expect(preserved.success).toBe(true);
|
|
expect(preserved.statusCode).toBe(200);
|
|
expect(preserved.condition).toBe("D");
|
|
} finally {
|
|
try { await unlink(testOutput); } catch(e) {}
|
|
}
|
|
});
|
|
|
|
});
|