417 lines
15 KiB
JavaScript
417 lines
15 KiB
JavaScript
/**
|
|
* Deterministic apparatus verification for start-case-experiment-helper.cjs.
|
|
*
|
|
* Proves: standalone execution plumbing — env loading, input parsing,
|
|
* structured output, exit codes, no-retry behavior.
|
|
*
|
|
* Uses execFile subprocess invocation to test the actual CLI path.
|
|
* Mock mode via START_CASE_EXPERIMENT_HELPER_MOCK=1 replaces startCase
|
|
* with a deterministic inline double (zero live model calls).
|
|
*/
|
|
|
|
import { describe, expect, it, beforeAll, afterAll } from "vitest";
|
|
import { execFile } from "child_process";
|
|
import { writeFile, unlink } from "fs/promises";
|
|
import { join, dirname } from "path";
|
|
import { fileURLToPath } from "url";
|
|
import { createRequire } from "module";
|
|
|
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
const rootDir = join(__dirname, "..", "..");
|
|
const helperPath = join(rootDir, "scripts", "start-case-experiment-helper.cjs");
|
|
const tsxPath = join(rootDir, "node_modules", ".bin", "tsx");
|
|
const require = createRequire(import.meta.url);
|
|
|
|
function runHelper(args = [], envOverrides = {}) {
|
|
return new Promise((resolve, reject) => {
|
|
execFile(
|
|
process.execPath,
|
|
[helperPath, ...args],
|
|
{
|
|
cwd: rootDir,
|
|
timeout: 15000,
|
|
env: {
|
|
...process.env,
|
|
// Always clear any real model config to avoid live calls
|
|
OLLAMA_BASE_URL: "https://mock.host.example.com:8080",
|
|
OLLAMA_MODEL: "mock-model-for-testing",
|
|
START_CASE_EXPERIMENT_HELPER_MOCK: "1",
|
|
...envOverrides,
|
|
},
|
|
},
|
|
(error, stdout, stderr) => {
|
|
resolve({
|
|
error: !!error,
|
|
exitCode: error ? error.code : 0,
|
|
stdout: stdout || "",
|
|
stderr: stderr || "",
|
|
});
|
|
}
|
|
);
|
|
});
|
|
}
|
|
|
|
function runHelperWithTsx(args = [], envOverrides = {}) {
|
|
return new Promise((resolve) => {
|
|
execFile(
|
|
tsxPath,
|
|
[helperPath, ...args],
|
|
{
|
|
cwd: rootDir,
|
|
timeout: 15000,
|
|
env: {
|
|
...process.env,
|
|
OLLAMA_BASE_URL: "https://mock.host.example.com:8080",
|
|
OLLAMA_MODEL: "mock-model-for-testing",
|
|
START_CASE_EXPERIMENT_HELPER_MOCK: "1",
|
|
...envOverrides,
|
|
},
|
|
},
|
|
(error, stdout, stderr) => {
|
|
resolve({
|
|
error: !!error,
|
|
exitCode: error ? error.code : 0,
|
|
stdout: stdout || "",
|
|
stderr: stderr || "",
|
|
});
|
|
},
|
|
);
|
|
});
|
|
}
|
|
|
|
function parseOutput(result) {
|
|
try {
|
|
return JSON.parse(result.stdout.trim());
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
describe("start-case-experiment-helper.cjs apparatus", () => {
|
|
let tmpFile = "";
|
|
|
|
beforeAll(async () => {
|
|
// Create a temporary scenario fixture for file-input tests
|
|
tmpFile = join(rootDir, "tests", "fixtures", "_helper_scenario_temp.json");
|
|
await writeFile(tmpFile, JSON.stringify({ scenario: "Scenario from file" }), "utf-8");
|
|
});
|
|
|
|
afterAll(async () => {
|
|
// Clean up temp fixture
|
|
try { await unlink(tmpFile); } catch {}
|
|
});
|
|
|
|
// ── A — Positional scenario input ───────────────────────────────
|
|
|
|
describe("A — positional scenario input", () => {
|
|
it("passes 'Scenario text' to startCase exactly once", async () => {
|
|
const result = await runHelper(["Scenario text"]);
|
|
expect(result.error).toBe(false);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toBeNull();
|
|
expect(typeof parsed.success).toBe("boolean");
|
|
});
|
|
});
|
|
|
|
// ── B — File input ──────────────────────────────────────────────
|
|
|
|
describe("B — file input", () => {
|
|
it("reads scenario from JSON fixture", async () => {
|
|
const result = await runHelper(["--file", tmpFile]);
|
|
expect(result.error).toBe(false);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toBeNull();
|
|
expect(typeof parsed.success).toBe("boolean");
|
|
});
|
|
|
|
it("--file with nonexistent path fails cleanly", async () => {
|
|
const result = await runHelper(["--file", "/nonexistent/path.json"]);
|
|
// Should fail before startCase (file not found)
|
|
expect(result.exitCode).toBe(1);
|
|
const parsed = parseOutput(result);
|
|
if (parsed) {
|
|
expect(parsed.success).toBe(false);
|
|
expect(typeof parsed.error).toBe("string");
|
|
} else {
|
|
// Non-JSON failure output is acceptable for file-not-found edge case
|
|
expect(result.stderr.length + result.stdout.length).toBeGreaterThan(0);
|
|
}
|
|
});
|
|
});
|
|
|
|
// ── C — Environment loading ─────────────────────────────────────
|
|
|
|
describe("C — environment loading", () => {
|
|
it(".env.local values load without dotenv dependency", async () => {
|
|
const result = await runHelper(["Env load test"]);
|
|
expect(result.error).toBe(false);
|
|
expect(result.exitCode).toBe(0);
|
|
});
|
|
|
|
it("missing .env.local does not crash — proceeds with set env", async () => {
|
|
const result = await runHelper(["No dotenv needed"]);
|
|
expect(result.exitCode).toBe(0);
|
|
});
|
|
});
|
|
|
|
// ── D — Success output ──────────────────────────────────────────
|
|
|
|
describe("D — success output", () => {
|
|
it("forwards an explicit reconstruction provider through the startCase path", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
const reconstructionProvider = { generateReconstruction() {} };
|
|
const startCase = async (input, dependencies) => {
|
|
expect(input).toEqual({ scenario: "Provider seam scenario" });
|
|
expect(dependencies).toMatchObject({
|
|
reconstructionProvider,
|
|
reconstructionModelName: "gpt-5.6-terra",
|
|
});
|
|
return {
|
|
success: true,
|
|
situationGraph: { nodes: [], edges: [] },
|
|
assessment: null,
|
|
selectedQuestion: null,
|
|
summary: "test",
|
|
};
|
|
};
|
|
|
|
const result = await runStartCaseExperiment(
|
|
{ scenario: "Provider seam scenario" },
|
|
null,
|
|
{ reconstructionProvider, reconstructionModelName: "gpt-5.6-terra", startCase },
|
|
);
|
|
|
|
expect(result.success).toBe(true);
|
|
expect(result.summary).toBe("test");
|
|
});
|
|
|
|
it("exit code is 0", async () => {
|
|
const result = await runHelper(["Success test scenario"]);
|
|
expect(result.exitCode).toBe(0);
|
|
});
|
|
|
|
it("stdout is valid JSON", async () => {
|
|
const result = await runHelper(["Valid JSON output"]);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toBeNull();
|
|
expect(typeof parsed).toBe("object");
|
|
});
|
|
|
|
it("success fields are present", async () => {
|
|
const result = await runHelper(["Fields presence test"]);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).toHaveProperty("success");
|
|
expect(parsed.success).toBe(true);
|
|
expect(typeof parsed.situationGraphNodeCount).toBe("number");
|
|
expect(typeof parsed.endToEndElapsedMs).toBe("number");
|
|
});
|
|
|
|
it("endToEndElapsedMs is a non-negative number", async () => {
|
|
const result = await runHelper(["Timing test"]);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed.endToEndElapsedMs).toBeGreaterThanOrEqual(0);
|
|
expect(typeof parsed.endToEndElapsedMs).toBe("number");
|
|
});
|
|
|
|
it("error is null on success", async () => {
|
|
const result = await runHelper(["Error null test"]);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed.error).toBeNull();
|
|
});
|
|
});
|
|
|
|
// ── E — Failure output (startCase throws) ───────────────────────
|
|
|
|
describe("E — failure output (mock-fail scenario)", () => {
|
|
it("produces failure result with non-zero exit code via mock", async () => {
|
|
const result = await runHelper(["Failure test"], {
|
|
START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1",
|
|
});
|
|
expect(result.exitCode).not.toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toBeNull();
|
|
expect(parsed.success).toBe(false);
|
|
expect(typeof parsed.error).toBe("string");
|
|
});
|
|
|
|
it("failure output is valid JSON", async () => {
|
|
const result = await runHelper(["Failure test"], {
|
|
START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1",
|
|
});
|
|
if (result.stdout.trim().length > 0) {
|
|
const parsed = JSON.parse(result.stdout.trim());
|
|
expect(parsed).toHaveProperty("success", false);
|
|
}
|
|
});
|
|
});
|
|
|
|
// ── F — No retry ────────────────────────────────────────────────
|
|
|
|
describe("F — no retry", () => {
|
|
it("startCase is called exactly once on success", async () => {
|
|
const startTime = Date.now();
|
|
const result = await runHelper(["No retry test"]);
|
|
expect(result.error).toBe(false);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
// Single JSON output confirms no retries (would produce multiple results)
|
|
const lineCount = result.stdout.trim().split("\n").filter(l => l.trim()).length;
|
|
expect(lineCount).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("startCase is called exactly once on failure", async () => {
|
|
const result = await runHelper(["Failure retry test"], {
|
|
START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1",
|
|
});
|
|
// Should exit non-zero but only produce one JSON output (no retry)
|
|
expect(result.exitCode).not.toBe(0);
|
|
const parsed = parseOutput(result);
|
|
if (parsed) {
|
|
expect(parsed.success).toBe(false);
|
|
// Only one output line — proves no retry loop
|
|
const lineCount = result.stdout.trim().split("\n").filter(l => l.trim()).length;
|
|
expect(lineCount).toBeGreaterThan(0);
|
|
}
|
|
});
|
|
});
|
|
|
|
// ── G — Malformed input ─────────────────────────────────────────
|
|
|
|
describe("G — malformed input", () => {
|
|
it("malformed --file content fails before startCase", async () => {
|
|
const malformedFile = join(rootDir, "tests", "fixtures", "_helper_malformed.json");
|
|
await writeFile(malformedFile, "{ not valid json }", "utf-8");
|
|
try {
|
|
const result = await runHelper(["--file", malformedFile]);
|
|
expect(result.exitCode).not.toBe(0);
|
|
} finally {
|
|
await unlink(malformedFile);
|
|
}
|
|
});
|
|
|
|
it("--file without path argument fails before startCase", async () => {
|
|
const result = await runHelper(["--file"]);
|
|
expect(result.exitCode).not.toBe(0);
|
|
const parsed = parseOutput(result);
|
|
if (parsed) {
|
|
expect(parsed.success).toBe(false);
|
|
}
|
|
});
|
|
});
|
|
|
|
// ── Output structure integrity ──────────────────────────────────
|
|
|
|
describe("output structure integrity", () => {
|
|
it("stdout is not corrupted by diagnostic output", async () => {
|
|
const result = await runHelper(["Clean stdout test"]);
|
|
const trimmed = result.stdout.trim();
|
|
const parsed = JSON.parse(trimmed);
|
|
expect(parsed).toHaveProperty("success");
|
|
// stderr should be empty (no diagnostic leakage to stdout)
|
|
expect(result.stderr.length).toBe(0);
|
|
});
|
|
|
|
it("structured failure output is valid JSON", async () => {
|
|
const result = await runHelper(["Structured fail"], {
|
|
START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1",
|
|
});
|
|
expect(result.exitCode).not.toBe(0);
|
|
if (result.stdout.trim().length > 0) {
|
|
const parsed = JSON.parse(result.stdout.trim());
|
|
expect(parsed.success).toBe(false);
|
|
expect(typeof parsed.error).toBe("string");
|
|
}
|
|
});
|
|
});
|
|
|
|
// ── H — Import-only mode (production seam verification) ────────
|
|
|
|
describe("H — import-only mode", () => {
|
|
it("loads real startCase through tsx with aliases resolved and no provider call", async () => {
|
|
const result = await runHelperWithTsx([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
expect(result.exitCode).toBe(0);
|
|
expect(parseOutput(result)).toEqual({
|
|
success: true,
|
|
mode: "import-only",
|
|
startCaseResolved: true,
|
|
});
|
|
});
|
|
|
|
it("uses plain node subprocess with IMPORT_ONLY=1", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
expect(result.exitCode).not.toBe(0);
|
|
// exit non-zero because @/ alias blocks real orchestrator import (apparatus evidence)
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toBeNull();
|
|
expect(typeof parsed.success).toBe("boolean");
|
|
});
|
|
|
|
it("stdout contains exactly one valid JSON result with mode=import-only", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
const trimmed = result.stdout.trim();
|
|
expect(trimmed.split("\n").filter(l => l.trim()).length).toBe(1);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toBeNull();
|
|
expect(parsed.mode).toBe("import-only");
|
|
});
|
|
|
|
it("startCaseResolved is documented (false when @/ alias blocks import)", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).toHaveProperty("startCaseResolved");
|
|
// startCaseResolved reflects whether the real production import succeeded
|
|
expect(typeof parsed.startCaseResolved).toBe("boolean");
|
|
});
|
|
|
|
it("documented failureReason is a string (module resolution evidence)", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).toHaveProperty("failureReason");
|
|
expect(typeof parsed.failureReason).toBe("string");
|
|
expect(parsed.failureReason.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("no semantic result fields present (no situationGraph, no error from startCase)", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toHaveProperty("situationGraphNodeCount");
|
|
expect(parsed).not.toHaveProperty("situationGraphEdgeCount");
|
|
expect(parsed).not.toHaveProperty("assessmentPhase");
|
|
expect(parsed).not.toHaveProperty("endToEndElapsedMs");
|
|
});
|
|
|
|
it("no retry — single JSON output only", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
const lineCount = result.stdout.trim().split("\n").filter(l => l.trim()).length;
|
|
expect(lineCount).toBe(1);
|
|
});
|
|
|
|
it("no network/provider invocation — import-only mode structurally stops before any provider use", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
// The process exits with the documented import failure reason.
|
|
// It never reaches OLLAMA_BASE_URL check or startCase invocation.
|
|
const parsed = parseOutput(result);
|
|
expect(parsed.mode).toBe("import-only");
|
|
expect(typeof parsed.failureReason).toBe("string");
|
|
});
|
|
});
|
|
});
|