658 lines
25 KiB
JavaScript
658 lines
25 KiB
JavaScript
/**
|
|
* Deterministic apparatus verification for start-case-experiment-helper.cjs.
|
|
*
|
|
* Proves: standalone execution plumbing — env loading, input parsing,
|
|
* structured output, exit codes, no-retry behavior.
|
|
*
|
|
* Uses execFile subprocess invocation to test the actual CLI path.
|
|
* Mock mode via START_CASE_EXPERIMENT_HELPER_MOCK=1 replaces startCase
|
|
* with a deterministic inline double (zero live model calls).
|
|
*/
|
|
|
|
import { describe, expect, it, beforeAll, afterAll, vi } from "vitest";
|
|
import { execFile } from "child_process";
|
|
import { readFile, writeFile, unlink } from "fs/promises";
|
|
import { join, dirname } from "path";
|
|
import { fileURLToPath } from "url";
|
|
import { createRequire } from "module";
|
|
|
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
const rootDir = join(__dirname, "..", "..");
|
|
const helperPath = join(rootDir, "scripts", "start-case-experiment-helper.cjs");
|
|
const tsxPath = join(rootDir, "node_modules", ".bin", "tsx");
|
|
const require = createRequire(import.meta.url);
|
|
|
|
function runHelper(args = [], envOverrides = {}) {
|
|
return new Promise((resolve, reject) => {
|
|
execFile(
|
|
process.execPath,
|
|
[helperPath, ...args],
|
|
{
|
|
cwd: rootDir,
|
|
timeout: 15000,
|
|
env: {
|
|
...process.env,
|
|
// Always clear any real model config to avoid live calls
|
|
OLLAMA_BASE_URL: "https://mock.host.example.com:8080",
|
|
OLLAMA_MODEL: "mock-model-for-testing",
|
|
START_CASE_EXPERIMENT_HELPER_MOCK: "1",
|
|
...envOverrides,
|
|
},
|
|
},
|
|
(error, stdout, stderr) => {
|
|
resolve({
|
|
error: !!error,
|
|
exitCode: error ? error.code : 0,
|
|
stdout: stdout || "",
|
|
stderr: stderr || "",
|
|
});
|
|
}
|
|
);
|
|
});
|
|
}
|
|
|
|
function runHelperWithTsx(args = [], envOverrides = {}) {
|
|
return new Promise((resolve) => {
|
|
execFile(
|
|
tsxPath,
|
|
[helperPath, ...args],
|
|
{
|
|
cwd: rootDir,
|
|
timeout: 15000,
|
|
env: {
|
|
...process.env,
|
|
OLLAMA_BASE_URL: "https://mock.host.example.com:8080",
|
|
OLLAMA_MODEL: "mock-model-for-testing",
|
|
START_CASE_EXPERIMENT_HELPER_MOCK: "1",
|
|
...envOverrides,
|
|
},
|
|
},
|
|
(error, stdout, stderr) => {
|
|
resolve({
|
|
error: !!error,
|
|
exitCode: error ? error.code : 0,
|
|
stdout: stdout || "",
|
|
stderr: stderr || "",
|
|
});
|
|
},
|
|
);
|
|
});
|
|
}
|
|
|
|
function parseOutput(result) {
|
|
try {
|
|
return JSON.parse(result.stdout.trim());
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
describe("start-case-experiment-helper.cjs apparatus", () => {
|
|
let tmpFile = "";
|
|
let metadataFile = "";
|
|
|
|
beforeAll(async () => {
|
|
// Create a temporary scenario fixture for file-input tests
|
|
tmpFile = join(rootDir, "tests", "fixtures", "_helper_scenario_temp.json");
|
|
await writeFile(tmpFile, JSON.stringify({ scenario: "Scenario from file" }), "utf-8");
|
|
metadataFile = join(rootDir, "tests", "fixtures", "_helper_semantic_scenario_temp.json");
|
|
await writeFile(
|
|
metadataFile,
|
|
JSON.stringify({
|
|
experimentId: "semantic-open-question-001",
|
|
scenarioId: "manufacturing-expansion-001",
|
|
scenario: "Scenario with semantic inspection metadata",
|
|
promptVersion: "v0.5",
|
|
}),
|
|
"utf-8",
|
|
);
|
|
});
|
|
|
|
afterAll(async () => {
|
|
// Clean up temp fixture
|
|
try { await unlink(tmpFile); } catch {}
|
|
try { await unlink(metadataFile); } catch {}
|
|
});
|
|
|
|
// ── A — Positional scenario input ───────────────────────────────
|
|
|
|
describe("A — positional scenario input", () => {
|
|
it("passes 'Scenario text' to startCase exactly once", async () => {
|
|
const result = await runHelper(["Scenario text"]);
|
|
expect(result.error).toBe(false);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toBeNull();
|
|
expect(typeof parsed.success).toBe("boolean");
|
|
});
|
|
});
|
|
|
|
// ── B — File input ──────────────────────────────────────────────
|
|
|
|
describe("B — file input", () => {
|
|
it("reads scenario from JSON fixture", async () => {
|
|
const result = await runHelper(["--file", tmpFile]);
|
|
expect(result.error).toBe(false);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toBeNull();
|
|
expect(typeof parsed.success).toBe("boolean");
|
|
});
|
|
|
|
it("--file with nonexistent path fails cleanly", async () => {
|
|
const result = await runHelper(["--file", "/nonexistent/path.json"]);
|
|
// Should fail before startCase (file not found)
|
|
expect(result.exitCode).toBe(1);
|
|
const parsed = parseOutput(result);
|
|
if (parsed) {
|
|
expect(parsed.success).toBe(false);
|
|
expect(typeof parsed.error).toBe("string");
|
|
} else {
|
|
// Non-JSON failure output is acceptable for file-not-found edge case
|
|
expect(result.stderr.length + result.stdout.length).toBeGreaterThan(0);
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("B.1 — reconstruction-only CLI option", () => {
|
|
it("passes reconstructionOnly through the startCase seam without selecting OpenAI", async () => {
|
|
const result = await runHelper(["--reconstruction-only", "Scenario text"]);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed.diagnostics).toMatchObject({
|
|
modelName: "inline-test-double",
|
|
reconstructionOnly: true,
|
|
});
|
|
});
|
|
|
|
it("composes reconstruction-only with file input", async () => {
|
|
const result = await runHelper(["--reconstruction-only", "--file", tmpFile]);
|
|
expect(result.exitCode).toBe(0);
|
|
expect(parseOutput(result).diagnostics.reconstructionOnly).toBe(true);
|
|
});
|
|
|
|
it("preserves experiment metadata and exposes semantic evidence", async () => {
|
|
const result = await runHelper(["--reconstruction-only", "--file", metadataFile]);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
|
|
expect(parsed.experimentId).toBe("semantic-open-question-001");
|
|
expect(parsed.scenarioId).toBe("manufacturing-expansion-001");
|
|
expect(parsed.execution).toEqual({
|
|
provider: "ollama",
|
|
model: "inline-test-double",
|
|
promptVersion: "v0.5",
|
|
});
|
|
expect(parsed.reconstruction.importantUnknowns).toEqual([
|
|
{
|
|
id: "unknown-1",
|
|
description: "Canonical uncertainty from the deterministic test double",
|
|
confidence: "medium",
|
|
question: "What would resolve the deterministic uncertainty?",
|
|
},
|
|
]);
|
|
expect(parsed.graphUnknownNodes).toEqual([
|
|
{
|
|
id: "graph-unknown-1",
|
|
label: "Canonical uncertainty",
|
|
description: "Canonical uncertainty from the deterministic test double",
|
|
openQuestion: "What would resolve the deterministic uncertainty?",
|
|
status: "unknown",
|
|
},
|
|
]);
|
|
expect(parsed.diagnostics).toMatchObject({
|
|
validationStatus: "mock",
|
|
presentationValidationStatus: "valid",
|
|
presentationValidationIssues: [],
|
|
graphReferenceValidation: { valid: true, errors: [] },
|
|
compatibilityApplied: false,
|
|
compatibilityChanges: [],
|
|
compatibilityWarnings: [],
|
|
});
|
|
});
|
|
|
|
it("passes only production-supported request fields and calls startCase once", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
const importantUnknowns = [
|
|
{
|
|
id: "unknown-a",
|
|
description: "Canonical uncertainty A",
|
|
confidence: "high",
|
|
question: "What resolves uncertainty A?",
|
|
},
|
|
];
|
|
const startCase = vi.fn(async () => ({
|
|
success: true,
|
|
reconstruction: { importantUnknowns },
|
|
situationGraph: {
|
|
nodes: [
|
|
{
|
|
id: "unknown-node-a",
|
|
label: "Canonical uncertainty A",
|
|
description: "Canonical uncertainty A",
|
|
openQuestion: "What resolves uncertainty A?",
|
|
status: "unknown",
|
|
kind: "unknown",
|
|
},
|
|
{
|
|
id: "observation-node-a",
|
|
label: "Known observation",
|
|
description: "Known observation",
|
|
status: "supported",
|
|
kind: "observation",
|
|
},
|
|
],
|
|
edges: [],
|
|
},
|
|
selectedQuestion: null,
|
|
summary: "test",
|
|
diagnostics: {
|
|
modelName: "configured-model",
|
|
promptVersion: "v0.5",
|
|
validationStatus: "valid",
|
|
presentationValidationStatus: "partial",
|
|
presentationValidationIssues: [{ category: "test-diagnostic" }],
|
|
graphReferenceValidation: { valid: true, errors: [] },
|
|
compatibilityApplied: true,
|
|
compatibilityChanges: ["normalised"],
|
|
compatibilityWarnings: ["warning"],
|
|
},
|
|
}));
|
|
|
|
const result = await runStartCaseExperiment(
|
|
{
|
|
experimentId: "experiment-a",
|
|
scenarioId: "scenario-a",
|
|
scenario: "Scenario A",
|
|
promptVersion: "v0.5",
|
|
ignoredMetadata: "must not cross the production boundary",
|
|
},
|
|
null,
|
|
{ startCase, reconstructionOnly: true },
|
|
);
|
|
|
|
expect(startCase).toHaveBeenCalledTimes(1);
|
|
expect(startCase).toHaveBeenCalledWith(
|
|
{ scenario: "Scenario A", promptVersion: "v0.5" },
|
|
expect.objectContaining({ reconstructionOnly: true }),
|
|
);
|
|
expect(result.experimentId).toBe("experiment-a");
|
|
expect(result.scenarioId).toBe("scenario-a");
|
|
expect(result.reconstruction.importantUnknowns).toBe(importantUnknowns);
|
|
expect(result.graphUnknownNodes).toEqual([
|
|
{
|
|
id: "unknown-node-a",
|
|
label: "Canonical uncertainty A",
|
|
description: "Canonical uncertainty A",
|
|
openQuestion: "What resolves uncertainty A?",
|
|
status: "unknown",
|
|
},
|
|
]);
|
|
expect(result.execution).toEqual({
|
|
provider: "ollama",
|
|
model: "configured-model",
|
|
promptVersion: "v0.5",
|
|
});
|
|
expect(result.diagnostics).toEqual({
|
|
modelName: "configured-model",
|
|
promptVersion: "v0.5",
|
|
validationStatus: "valid",
|
|
presentationValidationStatus: "partial",
|
|
presentationValidationIssues: [{ category: "test-diagnostic" }],
|
|
graphReferenceValidation: { valid: true, errors: [] },
|
|
compatibilityApplied: true,
|
|
compatibilityChanges: ["normalised"],
|
|
compatibilityWarnings: ["warning"],
|
|
});
|
|
});
|
|
});
|
|
|
|
// ── C — Environment loading ─────────────────────────────────────
|
|
|
|
describe("C — environment loading", () => {
|
|
it(".env.local values load without dotenv dependency", async () => {
|
|
const result = await runHelper(["Env load test"]);
|
|
expect(result.error).toBe(false);
|
|
expect(result.exitCode).toBe(0);
|
|
});
|
|
|
|
it("missing .env.local does not crash — proceeds with set env", async () => {
|
|
const result = await runHelper(["No dotenv needed"]);
|
|
expect(result.exitCode).toBe(0);
|
|
});
|
|
});
|
|
|
|
// ── D — Success output ──────────────────────────────────────────
|
|
|
|
describe("D — success output", () => {
|
|
it("selects the existing OpenAI provider through the helper injection seam", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
const previousProvider = process.env.START_CASE_EXPERIMENT_PROVIDER;
|
|
const previousKey = process.env.OPENAI_API_KEY;
|
|
process.env.START_CASE_EXPERIMENT_PROVIDER = "openai";
|
|
process.env.OPENAI_API_KEY = "test-key";
|
|
|
|
try {
|
|
const result = await runStartCaseExperiment(
|
|
{ scenario: "OpenAI provider selection scenario" },
|
|
null,
|
|
{
|
|
startCase: async (_input, dependencies) => {
|
|
expect(typeof dependencies.reconstructionProvider?.generateReconstruction).toBe("function");
|
|
expect(dependencies.reconstructionModelName).toBe("gpt-5.6-terra");
|
|
expect(dependencies.reconstructionOnly).toBe(true);
|
|
return {
|
|
success: true,
|
|
situationGraph: { nodes: [], edges: [] },
|
|
assessment: null,
|
|
selectedQuestion: null,
|
|
summary: "test",
|
|
};
|
|
},
|
|
},
|
|
);
|
|
|
|
expect(result.success).toBe(true);
|
|
} finally {
|
|
if (previousProvider === undefined) delete process.env.START_CASE_EXPERIMENT_PROVIDER;
|
|
else process.env.START_CASE_EXPERIMENT_PROVIDER = previousProvider;
|
|
if (previousKey === undefined) delete process.env.OPENAI_API_KEY;
|
|
else process.env.OPENAI_API_KEY = previousKey;
|
|
}
|
|
});
|
|
|
|
it("fails before startCase when OpenAI is selected without a key", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
const previousProvider = process.env.START_CASE_EXPERIMENT_PROVIDER;
|
|
const previousKey = process.env.OPENAI_API_KEY;
|
|
process.env.START_CASE_EXPERIMENT_PROVIDER = "openai";
|
|
delete process.env.OPENAI_API_KEY;
|
|
const startCase = vi.fn();
|
|
|
|
try {
|
|
await expect(
|
|
runStartCaseExperiment({ scenario: "Missing key scenario" }, null, { startCase }),
|
|
).rejects.toThrow("START_CASE_EXPERIMENT_PROVIDER=openai requires OPENAI_API_KEY");
|
|
expect(startCase).not.toHaveBeenCalled();
|
|
} finally {
|
|
if (previousProvider === undefined) delete process.env.START_CASE_EXPERIMENT_PROVIDER;
|
|
else process.env.START_CASE_EXPERIMENT_PROVIDER = previousProvider;
|
|
if (previousKey === undefined) delete process.env.OPENAI_API_KEY;
|
|
else process.env.OPENAI_API_KEY = previousKey;
|
|
}
|
|
});
|
|
|
|
it("forwards an explicit reconstruction provider through the startCase path", async () => {
|
|
const { runStartCaseExperiment } = require(helperPath);
|
|
const reconstructionProvider = { generateReconstruction() {} };
|
|
const startCase = async (input, dependencies) => {
|
|
expect(input).toEqual({ scenario: "Provider seam scenario" });
|
|
expect(dependencies).toMatchObject({
|
|
reconstructionProvider,
|
|
reconstructionModelName: "gpt-5.6-terra",
|
|
});
|
|
return {
|
|
success: true,
|
|
situationGraph: { nodes: [], edges: [] },
|
|
assessment: null,
|
|
selectedQuestion: null,
|
|
summary: "test",
|
|
};
|
|
};
|
|
|
|
const result = await runStartCaseExperiment(
|
|
{ scenario: "Provider seam scenario" },
|
|
null,
|
|
{ reconstructionProvider, reconstructionModelName: "gpt-5.6-terra", startCase },
|
|
);
|
|
|
|
expect(result.success).toBe(true);
|
|
expect(result.summary).toBe("test");
|
|
});
|
|
|
|
it("exit code is 0", async () => {
|
|
const result = await runHelper(["Success test scenario"]);
|
|
expect(result.exitCode).toBe(0);
|
|
});
|
|
|
|
it("stdout is valid JSON", async () => {
|
|
const result = await runHelper(["Valid JSON output"]);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toBeNull();
|
|
expect(typeof parsed).toBe("object");
|
|
});
|
|
|
|
it("success fields are present", async () => {
|
|
const result = await runHelper(["Fields presence test"]);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).toHaveProperty("success");
|
|
expect(parsed.success).toBe(true);
|
|
expect(typeof parsed.situationGraphNodeCount).toBe("number");
|
|
expect(typeof parsed.endToEndElapsedMs).toBe("number");
|
|
});
|
|
|
|
it("endToEndElapsedMs is a non-negative number", async () => {
|
|
const result = await runHelper(["Timing test"]);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed.endToEndElapsedMs).toBeGreaterThanOrEqual(0);
|
|
expect(typeof parsed.endToEndElapsedMs).toBe("number");
|
|
});
|
|
|
|
it("error is null on success", async () => {
|
|
const result = await runHelper(["Error null test"]);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed.error).toBeNull();
|
|
});
|
|
|
|
it("preserves the existing non-reconstruction-only output contract", async () => {
|
|
const result = await runHelper(["Compatibility scenario"]);
|
|
const parsed = parseOutput(result);
|
|
|
|
expect(parsed).not.toHaveProperty("experimentId");
|
|
expect(parsed).not.toHaveProperty("scenarioId");
|
|
expect(parsed).not.toHaveProperty("execution");
|
|
expect(parsed).not.toHaveProperty("graphUnknownNodes");
|
|
expect(parsed).toHaveProperty("situationGraphNodeCount");
|
|
expect(parsed).toHaveProperty("situationGraphEdgeCount");
|
|
expect(parsed).toHaveProperty("endToEndElapsedMs");
|
|
});
|
|
});
|
|
|
|
// ── E — Failure output (startCase throws) ───────────────────────
|
|
|
|
describe("E — failure output (mock-fail scenario)", () => {
|
|
it("produces failure result with non-zero exit code via mock", async () => {
|
|
const result = await runHelper(["Failure test"], {
|
|
START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1",
|
|
});
|
|
expect(result.exitCode).not.toBe(0);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toBeNull();
|
|
expect(parsed.success).toBe(false);
|
|
expect(typeof parsed.error).toBe("string");
|
|
});
|
|
|
|
it("failure output is valid JSON", async () => {
|
|
const result = await runHelper(["Failure test"], {
|
|
START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1",
|
|
});
|
|
if (result.stdout.trim().length > 0) {
|
|
const parsed = JSON.parse(result.stdout.trim());
|
|
expect(parsed).toHaveProperty("success", false);
|
|
}
|
|
});
|
|
});
|
|
|
|
// ── F — No retry ────────────────────────────────────────────────
|
|
|
|
describe("F — no retry", () => {
|
|
it("startCase is called exactly once on success", async () => {
|
|
const startTime = Date.now();
|
|
const result = await runHelper(["No retry test"]);
|
|
expect(result.error).toBe(false);
|
|
expect(result.exitCode).toBe(0);
|
|
const parsed = parseOutput(result);
|
|
// Single JSON output confirms no retries (would produce multiple results)
|
|
const lineCount = result.stdout.trim().split("\n").filter(l => l.trim()).length;
|
|
expect(lineCount).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("startCase is called exactly once on failure", async () => {
|
|
const result = await runHelper(["Failure retry test"], {
|
|
START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1",
|
|
});
|
|
// Should exit non-zero but only produce one JSON output (no retry)
|
|
expect(result.exitCode).not.toBe(0);
|
|
const parsed = parseOutput(result);
|
|
if (parsed) {
|
|
expect(parsed.success).toBe(false);
|
|
// Only one output line — proves no retry loop
|
|
const lineCount = result.stdout.trim().split("\n").filter(l => l.trim()).length;
|
|
expect(lineCount).toBeGreaterThan(0);
|
|
}
|
|
});
|
|
});
|
|
|
|
// ── G — Malformed input ─────────────────────────────────────────
|
|
|
|
describe("G — malformed input", () => {
|
|
it("malformed --file content fails before startCase", async () => {
|
|
const malformedFile = join(rootDir, "tests", "fixtures", "_helper_malformed.json");
|
|
await writeFile(malformedFile, "{ not valid json }", "utf-8");
|
|
try {
|
|
const result = await runHelper(["--file", malformedFile]);
|
|
expect(result.exitCode).not.toBe(0);
|
|
} finally {
|
|
await unlink(malformedFile);
|
|
}
|
|
});
|
|
|
|
it("--file without path argument fails before startCase", async () => {
|
|
const result = await runHelper(["--file"]);
|
|
expect(result.exitCode).not.toBe(0);
|
|
const parsed = parseOutput(result);
|
|
if (parsed) {
|
|
expect(parsed.success).toBe(false);
|
|
}
|
|
});
|
|
});
|
|
|
|
// ── Output structure integrity ──────────────────────────────────
|
|
|
|
describe("output structure integrity", () => {
|
|
it("stdout is not corrupted by diagnostic output", async () => {
|
|
const result = await runHelper(["Clean stdout test"]);
|
|
const trimmed = result.stdout.trim();
|
|
const parsed = JSON.parse(trimmed);
|
|
expect(parsed).toHaveProperty("success");
|
|
// stderr should be empty (no diagnostic leakage to stdout)
|
|
expect(result.stderr.length).toBe(0);
|
|
});
|
|
|
|
it("structured failure output is valid JSON", async () => {
|
|
const result = await runHelper(["Structured fail"], {
|
|
START_CASE_EXPERIMENT_HELPER_FORCE_FAIL: "1",
|
|
});
|
|
expect(result.exitCode).not.toBe(0);
|
|
if (result.stdout.trim().length > 0) {
|
|
const parsed = JSON.parse(result.stdout.trim());
|
|
expect(parsed.success).toBe(false);
|
|
expect(typeof parsed.error).toBe("string");
|
|
}
|
|
});
|
|
|
|
it("contains no investigation-persistence invocation", async () => {
|
|
const source = await readFile(helperPath, "utf-8");
|
|
expect(source).not.toMatch(/investigation-storage|server-investigation-persistence|saveInvestigation|\/api\/investigations/);
|
|
});
|
|
});
|
|
|
|
// ── H — Import-only mode (production seam verification) ────────
|
|
|
|
describe("H — import-only mode", () => {
|
|
it("loads real startCase through tsx with aliases resolved and no provider call", async () => {
|
|
const result = await runHelperWithTsx([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
expect(result.exitCode).toBe(0);
|
|
expect(parseOutput(result)).toEqual({
|
|
success: true,
|
|
mode: "import-only",
|
|
startCaseResolved: true,
|
|
});
|
|
});
|
|
|
|
it("uses plain node subprocess with IMPORT_ONLY=1", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
expect(result.exitCode).not.toBe(0);
|
|
// exit non-zero because @/ alias blocks real orchestrator import (apparatus evidence)
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toBeNull();
|
|
expect(typeof parsed.success).toBe("boolean");
|
|
});
|
|
|
|
it("stdout contains exactly one valid JSON result with mode=import-only", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
const trimmed = result.stdout.trim();
|
|
expect(trimmed.split("\n").filter(l => l.trim()).length).toBe(1);
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toBeNull();
|
|
expect(parsed.mode).toBe("import-only");
|
|
});
|
|
|
|
it("startCaseResolved is documented (false when @/ alias blocks import)", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).toHaveProperty("startCaseResolved");
|
|
// startCaseResolved reflects whether the real production import succeeded
|
|
expect(typeof parsed.startCaseResolved).toBe("boolean");
|
|
});
|
|
|
|
it("documented failureReason is a string (module resolution evidence)", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).toHaveProperty("failureReason");
|
|
expect(typeof parsed.failureReason).toBe("string");
|
|
expect(parsed.failureReason.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("no semantic result fields present (no situationGraph, no error from startCase)", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
const parsed = parseOutput(result);
|
|
expect(parsed).not.toHaveProperty("situationGraphNodeCount");
|
|
expect(parsed).not.toHaveProperty("situationGraphEdgeCount");
|
|
expect(parsed).not.toHaveProperty("assessmentPhase");
|
|
expect(parsed).not.toHaveProperty("endToEndElapsedMs");
|
|
});
|
|
|
|
it("no retry — single JSON output only", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
const lineCount = result.stdout.trim().split("\n").filter(l => l.trim()).length;
|
|
expect(lineCount).toBe(1);
|
|
});
|
|
|
|
it("no network/provider invocation — import-only mode structurally stops before any provider use", async () => {
|
|
const result = await runHelper([], {
|
|
START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY: "1",
|
|
});
|
|
// The process exits with the documented import failure reason.
|
|
// It never reaches OLLAMA_BASE_URL check or startCase invocation.
|
|
const parsed = parseOutput(result);
|
|
expect(parsed.mode).toBe("import-only");
|
|
expect(typeof parsed.failureReason).toBe("string");
|
|
});
|
|
});
|
|
});
|