/** * Deterministic apparatus verification for start-case-experiment-helper.cjs. * * Proves: standalone execution plumbing — env loading, input parsing, * structured output, exit codes, no-retry behavior. * * Uses execFile subprocess invocation to test the actual CLI path. * Mock mode via START_CASE_EXPERIMENT_HELPER_MOCK=1 replaces startCase * with a deterministic inline double (zero live model calls). */ import { describe, expect, it, beforeAll, afterAll, vi } from "vitest"; import { execFile } from "child_process"; import { readFile, writeFile, unlink } from "fs/promises"; import { join, dirname } from "path"; import { fileURLToPath } from "url"; import { createRequire } from "module"; const __dirname = dirname(fileURLToPath(import.meta.url)); const rootDir = join(__dirname, "..", ".."); const helperPath = join(rootDir, "scripts", "start-case-experiment-helper.cjs"); const tsxPath = join(rootDir, "node_modules", ".bin", "tsx"); const require = createRequire(import.meta.url); function runHelper(args = [], envOverrides = {}) { return new Promise((resolve, reject) => { execFile( process.execPath, [helperPath, ...args], { cwd: rootDir, timeout: 15000, env: { ...process.env, // Always clear any real model config to avoid live calls OLLAMA_BASE_URL: "https://mock.host.example.com:8080", OLLAMA_MODEL: "mock-model-for-testing", START_CASE_EXPERIMENT_HELPER_MOCK: "1", ...envOverrides, }, }, (error, stdout, stderr) => { resolve({ error: !!error, exitCode: error ? error.code : 0, stdout: stdout || "", stderr: stderr || "", }); } ); }); } function runHelperWithTsx(args = [], envOverrides = {}) { return new Promise((resolve) => { execFile( tsxPath, [helperPath, ...args], { cwd: rootDir, timeout: 15000, env: { ...process.env, OLLAMA_BASE_URL: "https://mock.host.example.com:8080", OLLAMA_MODEL: "mock-model-for-testing", START_CASE_EXPERIMENT_HELPER_MOCK: "1", ...envOverrides, }, }, (error, stdout, stderr) => { resolve({ error: !!error, exitCode: error ? error.code : 0, stdout: stdout || "", stderr: stderr || "", }); }, ); }); } function parseOutput(result) { try { return JSON.parse(result.stdout.trim()); } catch { return null; } } describe("start-case-experiment-helper.cjs apparatus", () => { let tmpFile = ""; let metadataFile = ""; beforeAll(async () => { // Create a temporary scenario fixture for file-input tests tmpFile = join(rootDir, "tests", "fixtures", "_helper_scenario_temp.json"); await writeFile(tmpFile, JSON.stringify({ scenario: "Scenario from file" }), "utf-8"); metadataFile = join(rootDir, "tests", "fixtures", "_helper_semantic_scenario_temp.json"); await writeFile( metadataFile, JSON.stringify({ experimentId: "semantic-open-question-001", scenarioId: "manufacturing-expansion-001", scenario: "Scenario with semantic inspection metadata", promptVersion: "v0.5", }), "utf-8", ); }); afterAll(async () => { // Clean up temp fixture try { await unlink(tmpFile); } catch {} try { await unlink(metadataFile); } catch {} }); // ── A — Positional scenario input ─────────────────────────────── describe("A — positional scenario input", () => { it("passes 'Scenario text' to startCase exactly once", async () => { const result = await runHelper(["Scenario text"]); expect(result.error).toBe(false); expect(result.exitCode).toBe(0); const parsed = parseOutput(result); expect(parsed).not.toBeNull(); expect(typeof parsed.success).toBe("boolean"); }); }); // ── B — File input ────────────────────────────────────────────── describe("B — file input", () => { it("reads scenario from JSON fixture", async () => { const result = await runHelper(["--file", tmpFile]); expect(result.error).toBe(false); expect(result.exitCode).toBe(0); const parsed = parseOutput(result); expect(parsed).not.toBeNull(); expect(typeof parsed.success).toBe("boolean"); }); it("--file with nonexistent path fails cleanly", async () => { const result = await runHelper(["--file", "/nonexistent/path.json"]); // Should fail before startCase (file not found) expect(result.exitCode).toBe(1); const parsed = parseOutput(result); if (parsed) { expect(parsed.success).toBe(false); expect(typeof parsed.error).toBe("string"); } else { // Non-JSON failure output is acceptable for file-not-found edge case expect(result.stderr.length + result.stdout.length).toBeGreaterThan(0); } }); }); describe("B.1 — reconstruction-only CLI option", () => { it("passes reconstructionOnly through the startCase seam without selecting OpenAI", async () => { const result = await runHelper(["--reconstruction-only", "Scenario text"]); expect(result.exitCode).toBe(0); const parsed = parseOutput(result); expect(parsed.diagnostics).toMatchObject({ modelName: "inline-test-double", reconstructionOnly: true, }); }); it("composes reconstruction-only with file input", async () => { const result = await runHelper(["--reconstruction-only", "--file", tmpFile]); expect(result.exitCode).toBe(0); expect(parseOutput(result).diagnostics.reconstructionOnly).toBe(true); }); it("preserves experiment metadata and exposes semantic evidence", async () => { const result = await runHelper(["--reconstruction-only", "--file", metadataFile]); expect(result.exitCode).toBe(0); const parsed = parseOutput(result); expect(parsed.experimentId).toBe("semantic-open-question-001"); expect(parsed.scenarioId).toBe("manufacturing-expansion-001"); expect(parsed.execution).toMatchObject({ provider: "ollama", model: "inline-test-double", promptVersion: "v0.5", }); expect(parsed.reconstruction.importantUnknowns).toEqual([ { id: "unknown-1", description: "Canonical uncertainty from the deterministic test double", confidence: "medium", question: "What would resolve the deterministic uncertainty?", }, ]); expect(parsed.graphUnknownNodes).toEqual([ { id: "graph-unknown-1", label: "Canonical uncertainty", description: "Canonical uncertainty from the deterministic test double", openQuestion: "What would resolve the deterministic uncertainty?", status: "unknown", }, ]); expect(parsed.diagnostics).toMatchObject({ validationStatus: "mock", presentationValidationStatus: "valid", presentationValidationIssues: [], graphReferenceValidation: { valid: true, errors: [] }, compatibilityApplied: false, compatibilityChanges: [], compatibilityWarnings: [], }); }); it("passes only production-supported request fields and calls startCase once", async () => { const { runStartCaseExperiment } = require(helperPath); const importantUnknowns = [ { id: "unknown-a", description: "Canonical uncertainty A", confidence: "high", question: "What resolves uncertainty A?", }, ]; const startCase = vi.fn(async () => ({ success: true, reconstruction: { importantUnknowns }, situationGraph: { nodes: [ { id: "unknown-node-a", label: "Canonical uncertainty A", description: "Canonical uncertainty A", openQuestion: "What resolves uncertainty A?", status: "unknown", kind: "unknown", }, { id: "observation-node-a", label: "Known observation", description: "Known observation", status: "supported", kind: "observation", }, ], edges: [], }, selectedQuestion: null, summary: "test", diagnostics: { modelName: "configured-model", promptVersion: "v0.5", validationStatus: "valid", presentationValidationStatus: "partial", presentationValidationIssues: [{ category: "test-diagnostic" }], graphReferenceValidation: { valid: true, errors: [] }, compatibilityApplied: true, compatibilityChanges: ["normalised"], compatibilityWarnings: ["warning"], }, })); const result = await runStartCaseExperiment( { experimentId: "experiment-a", scenarioId: "scenario-a", scenario: "Scenario A", promptVersion: "v0.5", ignoredMetadata: "must not cross the production boundary", }, null, { startCase, reconstructionOnly: true }, ); expect(startCase).toHaveBeenCalledTimes(1); expect(startCase).toHaveBeenCalledWith( { scenario: "Scenario A", promptVersion: "v0.5" }, expect.objectContaining({ reconstructionOnly: true }), ); expect(result.experimentId).toBe("experiment-a"); expect(result.scenarioId).toBe("scenario-a"); expect(result.reconstruction.importantUnknowns).toBe(importantUnknowns); expect(result.graphUnknownNodes).toEqual([ { id: "unknown-node-a", label: "Canonical uncertainty A", description: "Canonical uncertainty A", openQuestion: "What resolves uncertainty A?", status: "unknown", }, ]); expect(result.execution).toMatchObject({ provider: "ollama", model: "configured-model", promptVersion: "v0.5", }); expect(result.diagnostics).toEqual({ modelName: "configured-model", promptVersion: "v0.5", validationStatus: "valid", presentationValidationStatus: "partial", presentationValidationIssues: [{ category: "test-diagnostic" }], graphReferenceValidation: { valid: true, errors: [] }, compatibilityApplied: true, compatibilityChanges: ["normalised"], compatibilityWarnings: ["warning"], }); }); }); // ── C — Environment loading ───────────────────────────────────── describe("C — environment loading", () => { it(".env.local values load without dotenv dependency", async () => { const result = await runHelper(["Env load test"]); expect(result.error).toBe(false); expect(result.exitCode).toBe(0); }); it("missing .env.local does not crash — proceeds with set env", async () => { const result = await runHelper(["No dotenv needed"]); expect(result.exitCode).toBe(0); }); }); // ── D — Success output ────────────────────────────────────────── describe("D — success output", () => { it("selects the existing OpenAI provider through the helper injection seam", async () => { const { runStartCaseExperiment } = require(helperPath); const previousProvider = process.env.START_CASE_EXPERIMENT_PROVIDER; const previousKey = process.env.OPENAI_API_KEY; process.env.START_CASE_EXPERIMENT_PROVIDER = "openai"; process.env.OPENAI_API_KEY = "test-key"; try { const result = await runStartCaseExperiment( { scenario: "OpenAI provider selection scenario" }, null, { startCase: async (_input, dependencies) => { expect(typeof dependencies.reconstructionProvider?.generateReconstruction).toBe("function"); expect(dependencies.reconstructionModelName).toBe("gpt-5.6-terra"); expect(dependencies.reconstructionOnly).toBe(true); return { success: true, situationGraph: { nodes: [], edges: [] }, assessment: null, selectedQuestion: null, summary: "test", }; }, }, ); expect(result.success).toBe(true); } finally { if (previousProvider === undefined) delete process.env.START_CASE_EXPERIMENT_PROVIDER; else process.env.START_CASE_EXPERIMENT_PROVIDER = previousProvider; if (previousKey === undefined) delete process.env.OPENAI_API_KEY; else process.env.OPENAI_API_KEY = previousKey; } }); it("fails before startCase when OpenAI is selected without a key", async () => { const { runStartCaseExperiment } = require(helperPath); const previousProvider = process.env.START_CASE_EXPERIMENT_PROVIDER; const previousKey = process.env.OPENAI_API_KEY; process.env.START_CASE_EXPERIMENT_PROVIDER = "openai"; delete process.env.OPENAI_API_KEY; const startCase = vi.fn(); try { await expect( runStartCaseExperiment({ scenario: "Missing key scenario" }, null, { startCase }), ).rejects.toThrow("START_CASE_EXPERIMENT_PROVIDER=openai requires OPENAI_API_KEY"); expect(startCase).not.toHaveBeenCalled(); } finally { if (previousProvider === undefined) delete process.env.START_CASE_EXPERIMENT_PROVIDER; else process.env.START_CASE_EXPERIMENT_PROVIDER = previousProvider; if (previousKey === undefined) delete process.env.OPENAI_API_KEY; else process.env.OPENAI_API_KEY = previousKey; } }); it("forwards an explicit reconstruction provider through the startCase path", async () => { const { runStartCaseExperiment } = require(helperPath); const reconstructionProvider = { generateReconstruction() {} }; const startCase = async (input, dependencies) => { expect(input).toEqual({ scenario: "Provider seam scenario" }); expect(dependencies).toMatchObject({ reconstructionProvider, reconstructionModelName: "gpt-5.6-terra", }); return { success: true, situationGraph: { nodes: [], edges: [] }, assessment: null, selectedQuestion: null, summary: "test", }; }; const result = await runStartCaseExperiment( { scenario: "Provider seam scenario" }, null, { reconstructionProvider, reconstructionModelName: "gpt-5.6-terra", startCase }, ); expect(result.success).toBe(true); expect(result.summary).toBe("test"); }); it("exit code is 0", async () => { const result = await runHelper(["Success test scenario"]); expect(result.exitCode).toBe(0); }); it("stdout is valid JSON", async () => { const result = await runHelper(["Valid JSON output"]); const parsed = parseOutput(result); expect(parsed).not.toBeNull(); expect(typeof parsed).toBe("object"); }); it("success fields are present", async () => { const result = await runHelper(["Fields presence test"]); const parsed = parseOutput(result); expect(parsed).toHaveProperty("success"); expect(parsed.success).toBe(true); expect(typeof parsed.situationGraphNodeCount).toBe("number"); expect(typeof parsed.endToEndElapsedMs).toBe("number"); }); it("endToEndElapsedMs is a non-negative number", async () => { const result = await runHelper(["Timing test"]); const parsed = parseOutput(result); expect(parsed.endToEndElapsedMs).toBeGreaterThanOrEqual(0); expect(typeof parsed.endToEndElapsedMs).toBe("number"); }); it("error is null on success", async () => { const result = await runHelper(["Error null test"]); const parsed = parseOutput(result); expect(parsed.error).toBeNull(); }); it("preserves the existing non-reconstruction-only output contract", async () => { const result = await runHelper(["Compatibility scenario"]); const parsed = parseOutput(result); expect(parsed).not.toHaveProperty("experimentId"); expect(parsed).not.toHaveProperty("scenarioId"); expect(parsed).not.toHaveProperty("execution"); expect(parsed).not.toHaveProperty("graphUnknownNodes"); expect(parsed).toHaveProperty("situationGraphNodeCount"); expect(parsed).toHaveProperty("situationGraphEdgeCount"); expect(parsed).toHaveProperty("endToEndElapsedMs"); }); }); // ── E — Failure output (startCase throws) ─────────────────────── describe("E — failure output (mock-fail scenario)", () => { it("produces failure result with non-zero exit code via mock", async () => { const result = await runHelper(["Failure test"], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1", }); expect(result.exitCode).not.toBe(0); const parsed = parseOutput(result); expect(parsed).not.toBeNull(); expect(parsed.success).toBe(false); expect(typeof parsed.error).toBe("string"); }); it("failure output is valid JSON", async () => { const result = await runHelper(["Failure test"], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1", }); if (result.stdout.trim().length > 0) { const parsed = JSON.parse(result.stdout.trim()); expect(parsed).toHaveProperty("success", false); } }); }); // ── F — No retry ──────────────────────────────────────────────── describe("F — no retry", () => { it("startCase is called exactly once on success", async () => { const startTime = Date.now(); const result = await runHelper(["No retry test"]); expect(result.error).toBe(false); expect(result.exitCode).toBe(0); const parsed = parseOutput(result); // Single JSON output confirms no retries (would produce multiple results) const lineCount = result.stdout.trim().split("\n").filter(l => l.trim()).length; expect(lineCount).toBeGreaterThan(0); }); it("startCase is called exactly once on failure", async () => { const result = await runHelper(["Failure retry test"], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1", }); // Should exit non-zero but only produce one JSON output (no retry) expect(result.exitCode).not.toBe(0); const parsed = parseOutput(result); if (parsed) { expect(parsed.success).toBe(false); // Only one output line — proves no retry loop const lineCount = result.stdout.trim().split("\n").filter(l => l.trim()).length; expect(lineCount).toBeGreaterThan(0); } }); }); // ── G — Malformed input ───────────────────────────────────────── describe("G — malformed input", () => { it("malformed --file content fails before startCase", async () => { const malformedFile = join(rootDir, "tests", "fixtures", "_helper_malformed.json"); await writeFile(malformedFile, "{ not valid json }", "utf-8"); try { const result = await runHelper(["--file", malformedFile]); expect(result.exitCode).not.toBe(0); } finally { await unlink(malformedFile); } }); it("--file without path argument fails before startCase", async () => { const result = await runHelper(["--file"]); expect(result.exitCode).not.toBe(0); const parsed = parseOutput(result); if (parsed) { expect(parsed.success).toBe(false); } }); }); // ── Output structure integrity ────────────────────────────────── describe("output structure integrity", () => { it("stdout is not corrupted by diagnostic output", async () => { const result = await runHelper(["Clean stdout test"]); const trimmed = result.stdout.trim(); const parsed = JSON.parse(trimmed); expect(parsed).toHaveProperty("success"); // stderr should be empty (no diagnostic leakage to stdout) expect(result.stderr.length).toBe(0); }); it("structured failure output is valid JSON", async () => { const result = await runHelper(["Structured fail"], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1", }); expect(result.exitCode).not.toBe(0); if (result.stdout.trim().length > 0) { const parsed = JSON.parse(result.stdout.trim()); expect(parsed.success).toBe(false); expect(typeof parsed.error).toBe("string"); } }); it("contains no investigation-persistence invocation", async () => { const source = await readFile(helperPath, "utf-8"); expect(source).not.toMatch(/investigation-storage|server-investigation-persistence|saveInvestigation|\/api\/investigations/); }); }); // ── H — Import-only mode (production seam verification) ──────── describe("H — import-only mode", () => { it("loads real startCase through tsx with aliases resolved and no provider call", async () => { const result = await runHelperWithTsx([], { START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1", }); expect(result.exitCode).toBe(0); expect(parseOutput(result)).toEqual({ success: true, mode: "import-only", startCaseResolved: true, }); }); it("uses plain node subprocess with IMPORT_ONLY=1", async () => { const result = await runHelper([], { START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1", }); expect(result.exitCode).not.toBe(0); // exit non-zero because @/ alias blocks real orchestrator import (apparatus evidence) const parsed = parseOutput(result); expect(parsed).not.toBeNull(); expect(typeof parsed.success).toBe("boolean"); }); it("stdout contains exactly one valid JSON result with mode=import-only", async () => { const result = await runHelper([], { START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1", }); const trimmed = result.stdout.trim(); expect(trimmed.split("\n").filter(l => l.trim()).length).toBe(1); const parsed = parseOutput(result); expect(parsed).not.toBeNull(); expect(parsed.mode).toBe("import-only"); }); it("startCaseResolved is documented (false when @/ alias blocks import)", async () => { const result = await runHelper([], { START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1", }); const parsed = parseOutput(result); expect(parsed).toHaveProperty("startCaseResolved"); // startCaseResolved reflects whether the real production import succeeded expect(typeof parsed.startCaseResolved).toBe("boolean"); }); it("documented failureReason is a string (module resolution evidence)", async () => { const result = await runHelper([], { START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1", }); const parsed = parseOutput(result); expect(parsed).toHaveProperty("failureReason"); expect(typeof parsed.failureReason).toBe("string"); expect(parsed.failureReason.length).toBeGreaterThan(0); }); it("no semantic result fields present (no situationGraph, no error from startCase)", async () => { const result = await runHelper([], { START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1", }); const parsed = parseOutput(result); expect(parsed).not.toHaveProperty("situationGraphNodeCount"); expect(parsed).not.toHaveProperty("situationGraphEdgeCount"); expect(parsed).not.toHaveProperty("assessmentPhase"); expect(parsed).not.toHaveProperty("endToEndElapsedMs"); }); it("no retry — single JSON output only", async () => { const result = await runHelper([], { START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1", }); const lineCount = result.stdout.trim().split("\n").filter(l => l.trim()).length; expect(lineCount).toBe(1); }); it("no network/provider invocation — import-only mode structurally stops before any provider use", async () => { const result = await runHelper([], { START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1", }); // The process exits with the documented import failure reason. // It never reaches OLLAMA_BASE_URL check or startCase invocation. const parsed = parseOutput(result); expect(parsed.mode).toBe("import-only"); expect(typeof parsed.failureReason).toBe("string"); }); }); // ── I — Condition B schema variant apparatus ──────────────────── describe("I — Condition B schema variant apparatus", () => { it("--condition B passes generationSchema through the startCase seam", async () => { const { runStartCaseExperiment } = require(helperPath); let receivedDependencies; const result = await runStartCaseExperiment( { scenario: "Condition B test scenario" }, null, { condition: "B", startCase: async (_input, dependencies) => { receivedDependencies = dependencies; return { success: true, situationGraph: { nodes: [], edges: [] }, assessment: null, selectedQuestion: null, summary: "test", diagnostics: { validationStatus: "valid" }, }; }, }, ); expect(result.success).toBe(true); expect(receivedDependencies.generationSchema).toBeDefined(); // Condition B schema: importantUnknowns items should NOT have question property const iuSchema = receivedDependencies.generationSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items; if (iuSchema && iuSchema.properties) { expect(iuSchema.properties).not.toHaveProperty("question"); } }); it("--condition A uses production schema with question required", async () => { const { runStartCaseExperiment } = require(helperPath); let receivedDependencies; const result = await runStartCaseExperiment( { scenario: "Condition A test scenario" }, null, { condition: "A", startCase: async (_input, dependencies) => { receivedDependencies = dependencies; return { success: true, situationGraph: { nodes: [], edges: [] }, assessment: null, selectedQuestion: null, summary: "test", diagnostics: { validationStatus: "valid" }, }; }, }, ); expect(result.success).toBe(true); }); it("--condition B with mock produces valid structured output", async () => { const result = await runHelper(["--condition", "B", "Mock scenario"], {}); expect(result.exitCode).toBe(0); const parsed = parseOutput(result); expect(parsed.success).toBe(true); }); it("Condition B schema JSON has importantUnknowns WITHOUT question field", async () => { const { conditionBJsonSchema, conditionAJsonSchema } = require( join(rootDir, "lib", "reconstruction", "schema-variant.js"), ); const bIuProps = conditionBJsonSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items?.properties; const aIuProps = conditionAJsonSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items?.properties; expect(bIuProps).toHaveProperty("id"); expect(bIuProps).toHaveProperty("description"); expect(bIuProps).toHaveProperty("confidence"); expect(bIuProps).not.toHaveProperty("question"); // Verify Condition A (production) still HAS question expect(aIuProps).toHaveProperty("id"); expect(aIuProps).toHaveProperty("description"); expect(aIuProps).toHaveProperty("confidence"); expect(aIuProps).toHaveProperty("question"); }); it("Condition B prompt file exists and contains no question-generation language", async () => { const fs = require("fs"); const altPromptPath = join(rootDir, "prompts", "reconstruct-v0.5-b-noreq.md"); expect(fs.existsSync(altPromptPath)).toBe(true); const content = fs.readFileSync(altPromptPath, "utf8"); // The B prompt should NOT instruct model to generate questions for importantUnknowns expect(content).not.toMatch(/express that same uncertainty in \`question\`/i); expect(content).not.toMatch(/natural direct question/i); // But Condition A prompt does contain this language const prodPromptPath = join(rootDir, "prompts", "reconstruct-v0.5.md"); const prodContent = fs.readFileSync(prodPromptPath, "utf8"); expect(prodContent).toMatch(/express that same uncertainty in \`question\`/i); }); it("Condition B prompt has identical line count to Condition A prompt", async () => { const fs = require("fs"); const altLines = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-b-noreq.md"), "utf8").split("\n").length; const prodLines = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5.md"), "utf8").split("\n").length; expect(altLines).toBe(prodLines); }); it("Condition B prompt differs from Condition A only in Rule 7a and output format example", async () => { const fs = require("fs"); const altContent = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-b-noreq.md"), "utf8"); const prodContent = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5.md"), "utf8"); expect(altContent).not.toBe(prodContent); expect(altContent).toMatch(/express its canonical uncertainty meaning as a description in \`description\` with strict semantic fidelity/i); }); it("Condition B schema validates importantUnknowns WITHOUT question field required", async () => { const { conditionBJsonSchema } = require( join(rootDir, "lib", "reconstruction", "schema-variant.js"), ); const bIuItems = conditionBJsonSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items; expect(bIuItems).toBeDefined(); if (bIuItems && bIuItems.required) { expect(bIuItems.required).not.toContain("question"); } }); }); // ── J — Condition D treatment selector apparatus ──────────────── describe("J — Condition D prompt selection", () => { it("default Condition D (no --treatment) selects original deconstruct prompt", async () => { const { runStartCaseExperiment } = require(helperPath); let receivedScenario; await runStartCaseExperiment( { scenario: "Condition D default test" }, null, { condition: "D", startCase: async (input) => { receivedScenario = input.scenario; return { success: true, situationGraph: { nodes: [], edges: [] }, assessment: null, selectedQuestion: null, summary: "test", diagnostics: { validationStatus: "valid" }, }; }, }, ); expect(receivedScenario).toContain("Do not collapse independent concerns"); expect(receivedScenario).toContain("Do not create artificial uncertainties to increase their number."); }); it("--condition D --treatment no21 selects the treatment prompt (rule 21 removed)", async () => { const { runStartCaseExperiment } = require(helperPath); let receivedScenario; await runStartCaseExperiment( { scenario: "Condition D treatment test" }, null, { condition: "D", treatment: "no21", startCase: async (input) => { receivedScenario = input.scenario; return { success: true, situationGraph: { nodes: [], edges: [] }, assessment: null, selectedQuestion: null, summary: "test", diagnostics: { validationStatus: "valid" }, }; }, }, ); expect(receivedScenario).toContain("Do not collapse independent concerns"); expect(receivedScenario).not.toContain("Do not create artificial uncertainties to increase their number."); }); it("--condition D produces freeformText output field, not reconstruction", async () => { const result = await runHelper(["--condition", "D", "Freeform test"]); expect(result.exitCode).toBe(0); const parsed = parseOutput(result); expect(parsed).toHaveProperty("freeformText"); expect(parsed.conditionType).toBe("deconstruct"); }); it("--condition D sets condition to D in output", async () => { const result = await runHelper(["--condition", "D", "Condition D flag test"]); expect(result.exitCode).toBe(0); const parsed = parseOutput(result); expect(parsed.condition).toBe("D"); }); }); describe("J.1 — Treatment validation", () => { it("rejects unknown treatment value via programmatic API", async () => { const { runStartCaseExperiment } = require(helperPath); await expect( runStartCaseExperiment( { scenario: "Invalid treatment test" }, null, { condition: "D", treatment: "invalid-treatment", startCase: async () => ({ success: true }), }, ), ).rejects.toThrow("Invalid treatment: invalid-treatment. Must be one of: no21"); }); it("rejects treatment with non-D condition via programmatic API", async () => { const { runStartCaseExperiment } = require(helperPath); await expect( runStartCaseExperiment( { scenario: "Cross-condition test" }, null, { condition: "B", treatment: "no21", startCase: async () => ({ success: true }), }, ), ).rejects.toThrow('Treatment "no21" is only valid with --condition D or E, not --condition B'); }); it("rejects unknown condition value via CLI", async () => { const result = await runHelper(["--condition", "Z", "Invalid condition test"]); expect(result.exitCode).not.toBe(0); const parsed = parseOutput(result); if (parsed) { expect(parsed.success).toBe(false); expect(typeof parsed.error).toBe("string"); } }); it("rejects treatment with non-D condition via CLI", async () => { const result = await runHelper(["--condition", "B", "--treatment", "no21", "Cross-condition CLI test"]); expect(result.exitCode).not.toBe(0); const parsed = parseOutput(result); if (parsed) { expect(parsed.success).toBe(false); expect(typeof parsed.error).toBe("string"); } }); }); describe("J.2 — Prompt isolation verification", () => { it("original and treatment prompts differ only by removal of rule 21", async () => { const fs = require("fs"); const path = require("path"); const originalPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct.md"), "utf8"); const treatmentPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct-no21.md"), "utf8"); expect(originalPrompt).not.toBe(treatmentPrompt); // The only difference should be the removal of rule 21 line and its blank line const originalLines = originalPrompt.split("\n"); const treatmentLines = treatmentPrompt.split("\n"); // Find where they diverge — rule 21 is on line 21 expect(originalLines[20]).toBe("Do not create artificial uncertainties to increase their number."); // After removal, everything else should shift by exactly one line for (let i = 21; i < treatmentLines.length; i++) { expect(originalLines[i + 1]).toBe(treatmentLines[i]); } }); it("both prompts share identical preamble through rule 20", async () => { const fs = require("fs"); const originalPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct.md"), "utf8"); const treatmentPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct-no21.md"), "utf8"); // First 20 lines should be identical (before rule 21) const originalPreamble = originalPrompt.split("\n").slice(0, 20).join("\n"); const treatmentPreamble = treatmentPrompt.split("\n").slice(0, 20).join("\n"); expect(originalPreamble).toBe(treatmentPreamble); }); it("original prompt still contains rule 21 verbatim", async () => { const fs = require("fs"); const originalPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct.md"), "utf8"); expect(originalPrompt).toContain("Do not create artificial uncertainties to increase their number."); }); it("treatment prompt does NOT contain rule 21", async () => { const fs = require("fs"); const treatmentPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct-no21.md"), "utf8"); expect(treatmentPrompt).not.toContain("Do not create artificial uncertainties to increase their number."); }); it("Condition D output cannot overwrite original Condition D evidence — distinct freeformText field", async () => { const resultTreatment = await runHelper(["--condition", "D", "--treatment", "no21", "Treatment output test"]); const resultOriginal = await runHelper(["--condition", "D", "Original Condition D test"]); expect(resultTreatment.exitCode).toBe(0); expect(resultOriginal.exitCode).toBe(0); const treatmentOutput = parseOutput(resultTreatment); const originalOutput = parseOutput(resultOriginal); // Both should have freeformText (not reconstruction) for Condition D expect(treatmentOutput).toHaveProperty("freeformText"); expect(originalOutput).toHaveProperty("freeformText"); expect(treatmentOutput).not.toHaveProperty("reconstruction"); expect(originalOutput).not.toHaveProperty("reconstruction"); // They use the same output structure (freeformText) but different content — // the test double returns deterministic text that confirms prompt selection works expect(typeof treatmentOutput.freeformText).toBe("string"); expect(typeof originalOutput.freeformText).toBe("string"); }); }); // ── K — Condition E investigability-first apparatus ──────── describe("K — Condition E investigability-first apparatus", () => { it("--condition E selects the investigability-first prompt rather than deconstruct", async () => { const { runStartCaseExperiment } = require(helperPath); let receivedScenario; await runStartCaseExperiment( { scenario: "Condition E test" }, null, { condition: "E", startCase: async (input) => { receivedScenario = input.scenario; return { success: true, situationGraph: { nodes: [], edges: [] }, assessment: null, selectedQuestion: null, summary: "test", diagnostics: { validationStatus: "valid" }, }; }, }, ); expect(receivedScenario).toContain("You are a neutral analyst performing deconstruction of the user's situation"); expect(receivedScenario).toContain("Do not create artificial uncertainties to increase their number."); }); it("--condition E --treatment e-investigability-first is accepted", async () => { const { runStartCaseExperiment } = require(helperPath); await runStartCaseExperiment( { scenario: "Condition E treatment test" }, null, { condition: "E", treatment: "e-investigability-first", startCase: async (input) => { return { success: true, situationGraph: { nodes: [], edges: [] }, assessment: null, selectedQuestion: null, summary: "test", diagnostics: { validationStatus: "valid" }, }; }, }, ); // If we reach here without an error, the treatment was accepted }); it("--condition E produces freeformText output field via CLI mock", async () => { const result = await runHelper(["--condition", "E", "Freeform test"]); expect(result.exitCode).toBe(0); const parsed = parseOutput(result); expect(parsed).toHaveProperty("freeformText"); expect(typeof parsed.freeformText).toBe("string"); expect(parsed.conditionType).toBe("investigability-first"); }); it("--condition E sets condition to E in output", async () => { const result = await runHelper(["--condition", "E", "Condition E flag test"]); expect(result.exitCode).toBe(0); const parsed = parseOutput(result); expect(parsed.condition).toBe("E"); }); it("--condition E does not return reconstruction field in output", async () => { const result = await runHelper(["--condition", "E", "No reconstruction test"]); expect(result.exitCode).toBe(0); const parsed = parseOutput(result); expect(parsed).not.toHaveProperty("reconstruction"); }); it("--condition E --reconstruction-only sets execution.condition to E", async () => { const result = await runHelper(["--condition", "E", "--reconstruction-only", "Reconstruction only test"]); expect(result.exitCode).toBe(0); const parsed = parseOutput(result); expect(parsed.execution).toBeDefined(); expect(parsed.execution.condition).toBe("E"); }); it("--condition E rejects unknown treatment via programmatic API", async () => { const { runStartCaseExperiment } = require(helperPath); await expect( runStartCaseExperiment( { scenario: "Invalid treatment test" }, null, { condition: "E", treatment: "invalid-treatment", startCase: async () => ({ success: true }), }, ), ).rejects.toThrow("Invalid treatment: invalid-treatment. Must be one of: e-investigability-first"); }); it("--condition E rejects treatment with non-E condition via programmatic API", async () => { const { runStartCaseExperiment } = require(helperPath); await expect( runStartCaseExperiment( { scenario: "Cross-condition test" }, null, { condition: "B", treatment: "e-investigability-first", startCase: async () => ({ success: true }), }, ), ).rejects.toThrow('Treatment "e-investigability-first" is only valid with --condition D or E, not --condition B'); }); it("--condition E exposes non-empty freeformText (mock)", async () => { const result = await runHelper(["--condition", "E", "Freeform content test"]); expect(result.exitCode).toBe(0); const parsed = parseOutput(result); expect(parsed.freeformText.length).toBeGreaterThan(0); }); }); }); // ── Evidence persistence wrapper tests ───────────────────────────── var wrapperPath = join(rootDir, "scripts", "evidence-persist.cjs"); function runWrapper(args, envOverrides) { return new Promise(function(resolve) { execFile( process.execPath, [wrapperPath].concat(args), { cwd: rootDir, timeout: 30000, env: Object.assign({}, process.env, { START_CASE_EXPERIMENT_HELPER_MOCK: "1", }, envOverrides || {}), }, function(error, stdout, stderr) { resolve({ error: !!error, exitCode: error ? error.code : 0, stdout: stdout || "", stderr: stderr || "", }); } ); }); } /** * Tests for the evidence-persist.cjs wrapper. * * Uses START_CASE_EXPERIMENT_HELPER_MOCK=1 for deterministic behavior. */ describe("evidence persistence", function() { it("successful mock invocation publishes valid evidence with exact raw stdout bytes preserved", async function() { var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_d.json"); var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json"); try { var result = await runWrapper([ "--condition", "D", "--file", fixturePath, "--output", testOutput ]); expect(result.exitCode).toBe(0); expect(result.error).toBe(false); // Verify the evidence file was published var statSync = require("fs").statSync; var exists = false; try { statSync(testOutput); exists = true; } catch(e) {} expect(exists).toBe(true); // Verify the published file is valid JSON matching what helper outputs var rawContent = await readFile(testOutput, "utf-8"); var published = JSON.parse(rawContent.trim()); expect(published.success).toBe(true); expect(published.statusCode).toBe(200); expect(published.condition).toBe("D"); expect(typeof published.freeformText).toBe("string"); expect(published.freeformText.length).toBeGreaterThan(0); expect(published.diagnostics.reconstructionOnly).toBe(true); // Verify raw bytes are preserved (no reformatting) var reparseable = JSON.parse(rawContent.trim()); expect(reparseable.condition).toBe("D"); } finally { try { await unlink(testOutput); } catch(e2) {} } }); it("forced helper failure does not publish evidence", async function() { var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_fail.json"); var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json"); try { var result = await runWrapper([ "--condition", "D", "--file", fixturePath, "--output", testOutput ], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1" }); expect(result.exitCode).not.toBe(0); // Verify the final destination was NOT created var statSync = require("fs").statSync; var exists = false; try { statSync(testOutput); exists = true; } catch(e) {} expect(exists).toBe(false); } finally { try { await unlink(testOutput); } catch(e2) {} } }); it("malformed or incomplete output is rejected without publication", async function() { var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_malformed.json"); var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json"); try { var result = await runWrapper([ "--condition", "D", "--file", fixturePath, "--output", testOutput ], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1" }); expect(result.exitCode).not.toBe(0); // Verify destination was NOT created (rejection without publication) var statSync2 = require("fs").statSync; var exists = false; try { statSync2(testOutput); exists = true; } catch(e) {} expect(exists).toBe(false); } finally { try { await unlink(testOutput); } catch(e2) {} } }); it("an existing destination is preserved unchanged", async function() { var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_existing.json"); var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json"); try { // Pre-create the destination with known content await writeFile(testOutput, JSON.stringify({ preserved: true }), "utf-8"); var original = JSON.parse((await readFile(testOutput, "utf-8")).trim()); expect(original.preserved).toBe(true); // Attempt to publish — should be rejected due to existing destination var result = await runWrapper([ "--condition", "D", "--file", fixturePath, "--output", testOutput ]); expect(result.exitCode).not.toBe(0); // Verify the original content was NOT modified var final = JSON.parse((await readFile(testOutput, "utf-8")).trim()); expect(final.preserved).toBe(true); } finally { try { await unlink(testOutput); } catch(e2) {} } }); it("two competing publication attempts cannot both claim the same destination", async function() { var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_competing.json"); var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json"); try { // First publication — succeeds var result1 = await runWrapper([ "--condition", "D", "--file", fixturePath, "--output", testOutput ]); expect(result1.exitCode).toBe(0); expect(result1.error).toBe(false); // Verify winning evidence bytes var rawBytes1 = await readFile(testOutput, "utf-8"); var evidence1 = JSON.parse(rawBytes1.trim()); expect(evidence1.success).toBe(true); expect(evidence1.statusCode).toBe(200); expect(evidence1.condition).toBe("D"); // Simulate a competing claim: write different content to the same destination var competingContent = JSON.stringify({ overwrittenBy: "competing-claim" }); await writeFile(testOutput, competingContent, "utf-8"); // Second publication to the same path — must fail (EEXIST) var result2 = await runWrapper([ "--condition", "D", "--file", fixturePath, "--output", testOutput ]); expect(result2.exitCode).not.toBe(0); // Verify winning evidence bytes remain unchanged: // file still has the competing claim content (second wrapper did NOT overwrite it) var rawBytes2 = await readFile(testOutput, "utf-8"); expect(rawBytes2).toBe(competingContent); // Also verify first publication bytes are intact — they differ from current file, // confirming the second wrapper could not reach them expect(rawBytes2).not.toBe(rawBytes1); } finally { try { await unlink(testOutput); } catch(e2) {} } }); it("linkSync hardlink publication rejects EEXIST when a second process targets an already-published destination", async function() { var testOutput = join(rootDir, ".evidence-temp", "headless-semantic", "_test_persist_link_a.json"); var fixturePath = join(rootDir, "tests", "fixtures", "evidence-persist-input.json"); try { // First wrapper run — publishes successfully (linkSync creates hardlink at testOutput) var resultA = await runWrapper([ "--condition", "D", "--file", fixturePath, "--output", testOutput ]); expect(resultA.exitCode).toBe(0); // Verify the first publication produced valid evidence var publishedA = JSON.parse((await readFile(testOutput, "utf-8")).trim()); expect(publishedA.success).toBe(true); expect(publishedA.statusCode).toBe(200); expect(publishedA.condition).toBe("D"); expect(typeof publishedA.freeformText).toBe("string"); expect(publishedA.diagnostics.reconstructionOnly).toBe(true); // Second wrapper targets the exact same destination path. // linkSync(tmpFile, testOutput) must throw EEXIST because testOutput already exists // as a hardlink to the first publication's data (same inode). var resultB = await runWrapper([ "--condition", "D", "--file", fixturePath, "--output", testOutput ]); expect(resultB.exitCode).not.toBe(0); // Verify the first publication's evidence bytes remain exactly as published: // no-write guarantee confirmed — linkSync refuses to modify existing destination. var preserved = JSON.parse((await readFile(testOutput, "utf-8")).trim()); expect(preserved.success).toBe(true); expect(preserved.statusCode).toBe(200); expect(preserved.condition).toBe("D"); } finally { try { await unlink(testOutput); } catch(e) {} } }); });