diff --git a/scripts/start-case-experiment-helper.cjs b/scripts/start-case-experiment-helper.cjs index 1ea3c2b..f8de414 100755 --- a/scripts/start-case-experiment-helper.cjs +++ b/scripts/start-case-experiment-helper.cjs @@ -67,6 +67,12 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti let reconstructionModelName = options.reconstructionModelName; let reconstructionOnly = options.reconstructionOnly; const isMock = process.env.START_CASE_EXPERIMENT_HELPER_MOCK === "1"; + const productionRequest = { + scenario: scenarioInput?.scenario, + ...(scenarioInput?.promptVersion !== undefined + ? { promptVersion: scenarioInput.promptVersion } + : {}), + }; if (process.env.START_CASE_EXPERIMENT_PROVIDER === "openai") { if (!process.env.OPENAI_API_KEY) { @@ -95,13 +101,51 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti } return { success: true, - situationGraph: { nodes: [], edges: [], activeUnknownNodeId: null, resolvedNodeIds: [], currentSummary: "test" }, + reconstruction: { + importantUnknowns: [ + { + id: "unknown-1", + description: "Canonical uncertainty from the deterministic test double", + confidence: "medium", + question: "What would resolve the deterministic uncertainty?", + }, + ], + }, + situationGraph: { + nodes: [ + { + id: "graph-unknown-1", + label: "Canonical uncertainty", + description: "Canonical uncertainty from the deterministic test double", + openQuestion: "What would resolve the deterministic uncertainty?", + status: "unknown", + kind: "unknown", + }, + { + id: "graph-state-1", + label: "Known state", + description: "A non-unknown node", + status: "supported", + kind: "state", + }, + ], + edges: [], + activeUnknownNodeId: null, + resolvedNodeIds: [], + currentSummary: "test", + }, assessment: { phase: "initial", progress: 0 }, selectedQuestion: null, summary: null, diagnostics: { validationStatus: "mock", modelName: "inline-test-double", + promptVersion: productionRequest.promptVersion ?? "v0.5", + presentationValidationStatus: "valid", + presentationValidationIssues: [], + compatibilityApplied: false, + compatibilityChanges: [], + compatibilityWarnings: [], reconstructionOnly: dependencies.reconstructionOnly === true, graphReferenceValidation: { valid: true, errors: [] }, }, @@ -125,7 +169,7 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti let result; const endToEndStartedAt = Date.now(); try { - result = await startCase(scenarioInput, { + result = await startCase(productionRequest, { reconstructionProvider, reconstructionModelName, reconstructionOnly, @@ -147,7 +191,7 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti } } - return { + const output = { success: result.success, summary: result.summary ?? null, reconstruction: result.reconstruction ?? null, @@ -162,6 +206,47 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti error: result.success ? null : (result.error ?? "unknown"), statusCode: result.statusCode ?? (result.success ? 200 : 500), }; + + if (reconstructionOnly) { + if (scenarioInput?.experimentId !== undefined) { + output.experimentId = scenarioInput.experimentId; + } + if (scenarioInput?.scenarioId !== undefined) { + output.scenarioId = scenarioInput.scenarioId; + } + + const configuredProvider = + process.env.START_CASE_EXPERIMENT_PROVIDER === "openai" ? "openai" : "ollama"; + const configuredModel = + result.diagnostics?.modelName ?? + reconstructionModelName ?? + (configuredProvider === "ollama" ? process.env.OLLAMA_MODEL : undefined); + const effectivePromptVersion = result.diagnostics?.promptVersion; + + output.execution = { + provider: configuredProvider, + ...(configuredModel !== undefined ? { model: configuredModel } : {}), + ...(effectivePromptVersion !== undefined + ? { promptVersion: effectivePromptVersion } + : {}), + }; + + if (Array.isArray(result.situationGraph?.nodes)) { + output.graphUnknownNodes = result.situationGraph.nodes + .filter((node) => node.kind === "unknown") + .map((node) => ({ + id: node.id, + label: node.label, + description: node.description, + ...(node.openQuestion !== undefined + ? { openQuestion: node.openQuestion } + : {}), + ...(node.status !== undefined ? { status: node.status } : {}), + })); + } + } + + return output; } if (require.main === module) { diff --git a/tests/scripts/start-case-experiment-helper.test.js b/tests/scripts/start-case-experiment-helper.test.js index e3d778f..1db1b35 100644 --- a/tests/scripts/start-case-experiment-helper.test.js +++ b/tests/scripts/start-case-experiment-helper.test.js @@ -11,7 +11,7 @@ import { describe, expect, it, beforeAll, afterAll, vi } from "vitest"; import { execFile } from "child_process"; -import { writeFile, unlink } from "fs/promises"; +import { readFile, writeFile, unlink } from "fs/promises"; import { join, dirname } from "path"; import { fileURLToPath } from "url"; import { createRequire } from "module"; @@ -89,16 +89,29 @@ function parseOutput(result) { describe("start-case-experiment-helper.cjs apparatus", () => { let tmpFile = ""; + let metadataFile = ""; beforeAll(async () => { // Create a temporary scenario fixture for file-input tests tmpFile = join(rootDir, "tests", "fixtures", "_helper_scenario_temp.json"); await writeFile(tmpFile, JSON.stringify({ scenario: "Scenario from file" }), "utf-8"); + metadataFile = join(rootDir, "tests", "fixtures", "_helper_semantic_scenario_temp.json"); + await writeFile( + metadataFile, + JSON.stringify({ + experimentId: "semantic-open-question-001", + scenarioId: "manufacturing-expansion-001", + scenario: "Scenario with semantic inspection metadata", + promptVersion: "v0.5", + }), + "utf-8", + ); }); afterAll(async () => { // Clean up temp fixture try { await unlink(tmpFile); } catch {} + try { await unlink(metadataFile); } catch {} }); // ── A — Positional scenario input ─────────────────────────────── @@ -157,6 +170,141 @@ describe("start-case-experiment-helper.cjs apparatus", () => { expect(result.exitCode).toBe(0); expect(parseOutput(result).diagnostics.reconstructionOnly).toBe(true); }); + + it("preserves experiment metadata and exposes semantic evidence", async () => { + const result = await runHelper(["--reconstruction-only", "--file", metadataFile]); + expect(result.exitCode).toBe(0); + const parsed = parseOutput(result); + + expect(parsed.experimentId).toBe("semantic-open-question-001"); + expect(parsed.scenarioId).toBe("manufacturing-expansion-001"); + expect(parsed.execution).toEqual({ + provider: "ollama", + model: "inline-test-double", + promptVersion: "v0.5", + }); + expect(parsed.reconstruction.importantUnknowns).toEqual([ + { + id: "unknown-1", + description: "Canonical uncertainty from the deterministic test double", + confidence: "medium", + question: "What would resolve the deterministic uncertainty?", + }, + ]); + expect(parsed.graphUnknownNodes).toEqual([ + { + id: "graph-unknown-1", + label: "Canonical uncertainty", + description: "Canonical uncertainty from the deterministic test double", + openQuestion: "What would resolve the deterministic uncertainty?", + status: "unknown", + }, + ]); + expect(parsed.diagnostics).toMatchObject({ + validationStatus: "mock", + presentationValidationStatus: "valid", + presentationValidationIssues: [], + graphReferenceValidation: { valid: true, errors: [] }, + compatibilityApplied: false, + compatibilityChanges: [], + compatibilityWarnings: [], + }); + }); + + it("passes only production-supported request fields and calls startCase once", async () => { + const { runStartCaseExperiment } = require(helperPath); + const importantUnknowns = [ + { + id: "unknown-a", + description: "Canonical uncertainty A", + confidence: "high", + question: "What resolves uncertainty A?", + }, + ]; + const startCase = vi.fn(async () => ({ + success: true, + reconstruction: { importantUnknowns }, + situationGraph: { + nodes: [ + { + id: "unknown-node-a", + label: "Canonical uncertainty A", + description: "Canonical uncertainty A", + openQuestion: "What resolves uncertainty A?", + status: "unknown", + kind: "unknown", + }, + { + id: "observation-node-a", + label: "Known observation", + description: "Known observation", + status: "supported", + kind: "observation", + }, + ], + edges: [], + }, + selectedQuestion: null, + summary: "test", + diagnostics: { + modelName: "configured-model", + promptVersion: "v0.5", + validationStatus: "valid", + presentationValidationStatus: "partial", + presentationValidationIssues: [{ category: "test-diagnostic" }], + graphReferenceValidation: { valid: true, errors: [] }, + compatibilityApplied: true, + compatibilityChanges: ["normalised"], + compatibilityWarnings: ["warning"], + }, + })); + + const result = await runStartCaseExperiment( + { + experimentId: "experiment-a", + scenarioId: "scenario-a", + scenario: "Scenario A", + promptVersion: "v0.5", + ignoredMetadata: "must not cross the production boundary", + }, + null, + { startCase, reconstructionOnly: true }, + ); + + expect(startCase).toHaveBeenCalledTimes(1); + expect(startCase).toHaveBeenCalledWith( + { scenario: "Scenario A", promptVersion: "v0.5" }, + expect.objectContaining({ reconstructionOnly: true }), + ); + expect(result.experimentId).toBe("experiment-a"); + expect(result.scenarioId).toBe("scenario-a"); + expect(result.reconstruction.importantUnknowns).toBe(importantUnknowns); + expect(result.graphUnknownNodes).toEqual([ + { + id: "unknown-node-a", + label: "Canonical uncertainty A", + description: "Canonical uncertainty A", + openQuestion: "What resolves uncertainty A?", + status: "unknown", + }, + ]); + expect(result.execution).toEqual({ + provider: "ollama", + model: "configured-model", + promptVersion: "v0.5", + }); + expect(result.diagnostics).toEqual({ + modelName: "configured-model", + promptVersion: "v0.5", + validationStatus: "valid", + presentationValidationStatus: "partial", + presentationValidationIssues: [{ category: "test-diagnostic" }], + graphReferenceValidation: { valid: true, errors: [] }, + compatibilityApplied: true, + compatibilityChanges: ["normalised"], + compatibilityWarnings: ["warning"], + }); + }); }); // ── C — Environment loading ───────────────────────────────────── @@ -295,6 +443,19 @@ describe("start-case-experiment-helper.cjs apparatus", () => { const parsed = parseOutput(result); expect(parsed.error).toBeNull(); }); + + it("preserves the existing non-reconstruction-only output contract", async () => { + const result = await runHelper(["Compatibility scenario"]); + const parsed = parseOutput(result); + + expect(parsed).not.toHaveProperty("experimentId"); + expect(parsed).not.toHaveProperty("scenarioId"); + expect(parsed).not.toHaveProperty("execution"); + expect(parsed).not.toHaveProperty("graphUnknownNodes"); + expect(parsed).toHaveProperty("situationGraphNodeCount"); + expect(parsed).toHaveProperty("situationGraphEdgeCount"); + expect(parsed).toHaveProperty("endToEndElapsedMs"); + }); }); // ── E — Failure output (startCase throws) ─────────────────────── @@ -399,6 +560,11 @@ describe("start-case-experiment-helper.cjs apparatus", () => { expect(typeof parsed.error).toBe("string"); } }); + + it("contains no investigation-persistence invocation", async () => { + const source = await readFile(helperPath, "utf-8"); + expect(source).not.toMatch(/investigation-storage|server-investigation-persistence|saveInvestigation|\/api\/investigations/); + }); }); // ── H — Import-only mode (production seam verification) ────────