/** * Deterministic apparatus verification for start-case-experiment-helper.cjs. * * Proves: standalone execution plumbing — env loading, input parsing, * structured output, exit codes, no-retry behavior. * * Uses execFile subprocess invocation to test the actual CLI path. * Mock mode via START_CASE_EXPERIMENT_HELPER_MOCK=1 replaces startCase * with a deterministic inline double (zero live model calls). */ import { describe, expect, it, beforeAll, afterAll, vi } from "vitest"; import { execFile } from "child_process"; import { writeFile, unlink } from "fs/promises"; import { join, dirname } from "path"; import { fileURLToPath } from "url"; import { createRequire } from "module"; const __dirname = dirname(fileURLToPath(import.meta.url)); const rootDir = join(__dirname, "..", ".."); const helperPath = join(rootDir, "scripts", "start-case-experiment-helper.cjs"); const tsxPath = join(rootDir, "node_modules", ".bin", "tsx"); const require = createRequire(import.meta.url); function runHelper(args = [], envOverrides = {}) { return new Promise((resolve, reject) => { execFile( process.execPath, [helperPath, ...args], { cwd: rootDir, timeout: 15000, env: { ...process.env, // Always clear any real model config to avoid live calls OLLAMA_BASE_URL: "https://mock.host.example.com:8080", OLLAMA_MODEL: "mock-model-for-testing", START_CASE_EXPERIMENT_HELPER_MOCK: "1", ...envOverrides, }, }, (error, stdout, stderr) => { resolve({ error: !!error, exitCode: error ? error.code : 0, stdout: stdout || "", stderr: stderr || "", }); } ); }); } function runHelperWithTsx(args = [], envOverrides = {}) { return new Promise((resolve) => { execFile( tsxPath, [helperPath, ...args], { cwd: rootDir, timeout: 15000, env: { ...process.env, OLLAMA_BASE_URL: "https://mock.host.example.com:8080", OLLAMA_MODEL: "mock-model-for-testing", START_CASE_EXPERIMENT_HELPER_MOCK: "1", ...envOverrides, }, }, (error, stdout, stderr) => { resolve({ error: !!error, exitCode: error ? error.code : 0, stdout: stdout || "", stderr: stderr || "", }); }, ); }); } function parseOutput(result) { try { return JSON.parse(result.stdout.trim()); } catch { return null; } } describe("start-case-experiment-helper.cjs apparatus", () => { let tmpFile = ""; beforeAll(async () => { // Create a temporary scenario fixture for file-input tests tmpFile = join(rootDir, "tests", "fixtures", "_helper_scenario_temp.json"); await writeFile(tmpFile, JSON.stringify({ scenario: "Scenario from file" }), "utf-8"); }); afterAll(async () => { // Clean up temp fixture try { await unlink(tmpFile); } catch {} }); // ── A — Positional scenario input ─────────────────────────────── describe("A — positional scenario input", () => { it("passes 'Scenario text' to startCase exactly once", async () => { const result = await runHelper(["Scenario text"]); expect(result.error).toBe(false); expect(result.exitCode).toBe(0); const parsed = parseOutput(result); expect(parsed).not.toBeNull(); expect(typeof parsed.success).toBe("boolean"); }); }); // ── B — File input ────────────────────────────────────────────── describe("B — file input", () => { it("reads scenario from JSON fixture", async () => { const result = await runHelper(["--file", tmpFile]); expect(result.error).toBe(false); expect(result.exitCode).toBe(0); const parsed = parseOutput(result); expect(parsed).not.toBeNull(); expect(typeof parsed.success).toBe("boolean"); }); it("--file with nonexistent path fails cleanly", async () => { const result = await runHelper(["--file", "/nonexistent/path.json"]); // Should fail before startCase (file not found) expect(result.exitCode).toBe(1); const parsed = parseOutput(result); if (parsed) { expect(parsed.success).toBe(false); expect(typeof parsed.error).toBe("string"); } else { // Non-JSON failure output is acceptable for file-not-found edge case expect(result.stderr.length + result.stdout.length).toBeGreaterThan(0); } }); }); describe("B.1 — reconstruction-only CLI option", () => { it("passes reconstructionOnly through the startCase seam without selecting OpenAI", async () => { const result = await runHelper(["--reconstruction-only", "Scenario text"]); expect(result.exitCode).toBe(0); const parsed = parseOutput(result); expect(parsed.diagnostics).toMatchObject({ modelName: "inline-test-double", reconstructionOnly: true, }); }); it("composes reconstruction-only with file input", async () => { const result = await runHelper(["--reconstruction-only", "--file", tmpFile]); expect(result.exitCode).toBe(0); expect(parseOutput(result).diagnostics.reconstructionOnly).toBe(true); }); }); // ── C — Environment loading ───────────────────────────────────── describe("C — environment loading", () => { it(".env.local values load without dotenv dependency", async () => { const result = await runHelper(["Env load test"]); expect(result.error).toBe(false); expect(result.exitCode).toBe(0); }); it("missing .env.local does not crash — proceeds with set env", async () => { const result = await runHelper(["No dotenv needed"]); expect(result.exitCode).toBe(0); }); }); // ── D — Success output ────────────────────────────────────────── describe("D — success output", () => { it("selects the existing OpenAI provider through the helper injection seam", async () => { const { runStartCaseExperiment } = require(helperPath); const previousProvider = process.env.START_CASE_EXPERIMENT_PROVIDER; const previousKey = process.env.OPENAI_API_KEY; process.env.START_CASE_EXPERIMENT_PROVIDER = "openai"; process.env.OPENAI_API_KEY = "test-key"; try { const result = await runStartCaseExperiment( { scenario: "OpenAI provider selection scenario" }, null, { startCase: async (_input, dependencies) => { expect(typeof dependencies.reconstructionProvider?.generateReconstruction).toBe("function"); expect(dependencies.reconstructionModelName).toBe("gpt-5.6-terra"); expect(dependencies.reconstructionOnly).toBe(true); return { success: true, situationGraph: { nodes: [], edges: [] }, assessment: null, selectedQuestion: null, summary: "test", }; }, }, ); expect(result.success).toBe(true); } finally { if (previousProvider === undefined) delete process.env.START_CASE_EXPERIMENT_PROVIDER; else process.env.START_CASE_EXPERIMENT_PROVIDER = previousProvider; if (previousKey === undefined) delete process.env.OPENAI_API_KEY; else process.env.OPENAI_API_KEY = previousKey; } }); it("fails before startCase when OpenAI is selected without a key", async () => { const { runStartCaseExperiment } = require(helperPath); const previousProvider = process.env.START_CASE_EXPERIMENT_PROVIDER; const previousKey = process.env.OPENAI_API_KEY; process.env.START_CASE_EXPERIMENT_PROVIDER = "openai"; delete process.env.OPENAI_API_KEY; const startCase = vi.fn(); try { await expect( runStartCaseExperiment({ scenario: "Missing key scenario" }, null, { startCase }), ).rejects.toThrow("START_CASE_EXPERIMENT_PROVIDER=openai requires OPENAI_API_KEY"); expect(startCase).not.toHaveBeenCalled(); } finally { if (previousProvider === undefined) delete process.env.START_CASE_EXPERIMENT_PROVIDER; else process.env.START_CASE_EXPERIMENT_PROVIDER = previousProvider; if (previousKey === undefined) delete process.env.OPENAI_API_KEY; else process.env.OPENAI_API_KEY = previousKey; } }); it("forwards an explicit reconstruction provider through the startCase path", async () => { const { runStartCaseExperiment } = require(helperPath); const reconstructionProvider = { generateReconstruction() {} }; const startCase = async (input, dependencies) => { expect(input).toEqual({ scenario: "Provider seam scenario" }); expect(dependencies).toMatchObject({ reconstructionProvider, reconstructionModelName: "gpt-5.6-terra", }); return { success: true, situationGraph: { nodes: [], edges: [] }, assessment: null, selectedQuestion: null, summary: "test", }; }; const result = await runStartCaseExperiment( { scenario: "Provider seam scenario" }, null, { reconstructionProvider, reconstructionModelName: "gpt-5.6-terra", startCase }, ); expect(result.success).toBe(true); expect(result.summary).toBe("test"); }); it("exit code is 0", async () => { const result = await runHelper(["Success test scenario"]); expect(result.exitCode).toBe(0); }); it("stdout is valid JSON", async () => { const result = await runHelper(["Valid JSON output"]); const parsed = parseOutput(result); expect(parsed).not.toBeNull(); expect(typeof parsed).toBe("object"); }); it("success fields are present", async () => { const result = await runHelper(["Fields presence test"]); const parsed = parseOutput(result); expect(parsed).toHaveProperty("success"); expect(parsed.success).toBe(true); expect(typeof parsed.situationGraphNodeCount).toBe("number"); expect(typeof parsed.endToEndElapsedMs).toBe("number"); }); it("endToEndElapsedMs is a non-negative number", async () => { const result = await runHelper(["Timing test"]); const parsed = parseOutput(result); expect(parsed.endToEndElapsedMs).toBeGreaterThanOrEqual(0); expect(typeof parsed.endToEndElapsedMs).toBe("number"); }); it("error is null on success", async () => { const result = await runHelper(["Error null test"]); const parsed = parseOutput(result); expect(parsed.error).toBeNull(); }); }); // ── E — Failure output (startCase throws) ─────────────────────── describe("E — failure output (mock-fail scenario)", () => { it("produces failure result with non-zero exit code via mock", async () => { const result = await runHelper(["Failure test"], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1", }); expect(result.exitCode).not.toBe(0); const parsed = parseOutput(result); expect(parsed).not.toBeNull(); expect(parsed.success).toBe(false); expect(typeof parsed.error).toBe("string"); }); it("failure output is valid JSON", async () => { const result = await runHelper(["Failure test"], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1", }); if (result.stdout.trim().length > 0) { const parsed = JSON.parse(result.stdout.trim()); expect(parsed).toHaveProperty("success", false); } }); }); // ── F — No retry ──────────────────────────────────────────────── describe("F — no retry", () => { it("startCase is called exactly once on success", async () => { const startTime = Date.now(); const result = await runHelper(["No retry test"]); expect(result.error).toBe(false); expect(result.exitCode).toBe(0); const parsed = parseOutput(result); // Single JSON output confirms no retries (would produce multiple results) const lineCount = result.stdout.trim().split("\n").filter(l => l.trim()).length; expect(lineCount).toBeGreaterThan(0); }); it("startCase is called exactly once on failure", async () => { const result = await runHelper(["Failure retry test"], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1", }); // Should exit non-zero but only produce one JSON output (no retry) expect(result.exitCode).not.toBe(0); const parsed = parseOutput(result); if (parsed) { expect(parsed.success).toBe(false); // Only one output line — proves no retry loop const lineCount = result.stdout.trim().split("\n").filter(l => l.trim()).length; expect(lineCount).toBeGreaterThan(0); } }); }); // ── G — Malformed input ───────────────────────────────────────── describe("G — malformed input", () => { it("malformed --file content fails before startCase", async () => { const malformedFile = join(rootDir, "tests", "fixtures", "_helper_malformed.json"); await writeFile(malformedFile, "{ not valid json }", "utf-8"); try { const result = await runHelper(["--file", malformedFile]); expect(result.exitCode).not.toBe(0); } finally { await unlink(malformedFile); } }); it("--file without path argument fails before startCase", async () => { const result = await runHelper(["--file"]); expect(result.exitCode).not.toBe(0); const parsed = parseOutput(result); if (parsed) { expect(parsed.success).toBe(false); } }); }); // ── Output structure integrity ────────────────────────────────── describe("output structure integrity", () => { it("stdout is not corrupted by diagnostic output", async () => { const result = await runHelper(["Clean stdout test"]); const trimmed = result.stdout.trim(); const parsed = JSON.parse(trimmed); expect(parsed).toHaveProperty("success"); // stderr should be empty (no diagnostic leakage to stdout) expect(result.stderr.length).toBe(0); }); it("structured failure output is valid JSON", async () => { const result = await runHelper(["Structured fail"], { START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1", }); expect(result.exitCode).not.toBe(0); if (result.stdout.trim().length > 0) { const parsed = JSON.parse(result.stdout.trim()); expect(parsed.success).toBe(false); expect(typeof parsed.error).toBe("string"); } }); }); // ── H — Import-only mode (production seam verification) ──────── describe("H — import-only mode", () => { it("loads real startCase through tsx with aliases resolved and no provider call", async () => { const result = await runHelperWithTsx([], { START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1", }); expect(result.exitCode).toBe(0); expect(parseOutput(result)).toEqual({ success: true, mode: "import-only", startCaseResolved: true, }); }); it("uses plain node subprocess with IMPORT_ONLY=1", async () => { const result = await runHelper([], { START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1", }); expect(result.exitCode).not.toBe(0); // exit non-zero because @/ alias blocks real orchestrator import (apparatus evidence) const parsed = parseOutput(result); expect(parsed).not.toBeNull(); expect(typeof parsed.success).toBe("boolean"); }); it("stdout contains exactly one valid JSON result with mode=import-only", async () => { const result = await runHelper([], { START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1", }); const trimmed = result.stdout.trim(); expect(trimmed.split("\n").filter(l => l.trim()).length).toBe(1); const parsed = parseOutput(result); expect(parsed).not.toBeNull(); expect(parsed.mode).toBe("import-only"); }); it("startCaseResolved is documented (false when @/ alias blocks import)", async () => { const result = await runHelper([], { START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1", }); const parsed = parseOutput(result); expect(parsed).toHaveProperty("startCaseResolved"); // startCaseResolved reflects whether the real production import succeeded expect(typeof parsed.startCaseResolved).toBe("boolean"); }); it("documented failureReason is a string (module resolution evidence)", async () => { const result = await runHelper([], { START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1", }); const parsed = parseOutput(result); expect(parsed).toHaveProperty("failureReason"); expect(typeof parsed.failureReason).toBe("string"); expect(parsed.failureReason.length).toBeGreaterThan(0); }); it("no semantic result fields present (no situationGraph, no error from startCase)", async () => { const result = await runHelper([], { START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1", }); const parsed = parseOutput(result); expect(parsed).not.toHaveProperty("situationGraphNodeCount"); expect(parsed).not.toHaveProperty("situationGraphEdgeCount"); expect(parsed).not.toHaveProperty("assessmentPhase"); expect(parsed).not.toHaveProperty("endToEndElapsedMs"); }); it("no retry — single JSON output only", async () => { const result = await runHelper([], { START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1", }); const lineCount = result.stdout.trim().split("\n").filter(l => l.trim()).length; expect(lineCount).toBe(1); }); it("no network/provider invocation — import-only mode structurally stops before any provider use", async () => { const result = await runHelper([], { START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1", }); // The process exits with the documented import failure reason. // It never reaches OLLAMA_BASE_URL check or startCase invocation. const parsed = parseOutput(result); expect(parsed.mode).toBe("import-only"); expect(typeof parsed.failureReason).toBe("string"); }); }); });