#!/usr/bin/env node /** * Minimal startCase (initial decomposition) experiment harness. * * Loads .env.local, invokes startCase() through the real production path, * and prints a structured result for semantic evaluation. * * Usage: * node scripts/start-case-experiment-helper.cjs "" * node scripts/start-case-experiment-helper.cjs --file scenario.json * node scripts/start-case-experiment-helper.cjs --reconstruction-only --file scenario.json * * Exit codes: * 0 — success (structured result printed to stdout) * 1 — execution failure or invalid input */ // Use the standard environment loader already available to this Next repository. // loadEnvConfig mutates process.env in-place (same semantics as previous bespoke parser). const path = require("path"); const PROJECT_ROOT = path.resolve(__dirname, ".."); (function loadEnvironment() { try { const { loadEnvConfig } = require("@next/env"); loadEnvConfig(PROJECT_ROOT, undefined, { logOutput: "none" }, false); } catch { // loader unavailable — proceed with whatever is already set. } })(); function assertRequiredEnv(name) { const value = process.env[name]; if (!value) { throw new Error( `Live experiment requires ${name}. Set it in .env.local.\n` + `Found: OLLAMA_BASE_URL=${process.env.OLLAMA_BASE_URL ?? "(missing)"}, ` + `OLLAMA_MODEL=${process.env.OLLAMA_MODEL ?? "(missing)"}` ); } return value; } function readScenarioInput(argv) { if (argv.includes("--file")) { const idx = argv.indexOf("--file"); if (idx + 1 >= argv.length) throw new Error("--file requires a path argument"); const fs = require("fs"); const path = argv[idx + 1]; return JSON.parse(fs.readFileSync(path, "utf-8")); } // Use all positional args as the scenario text (join with space) const startIdx = argv.findIndex(a => a !== "" && !a.startsWith("-") && a !== "--"); if (startIdx === -1 || startIdx >= argv.length) throw new Error("Usage: node scripts/start-case-experiment-helper.cjs \"\" [--file path.json]"); return { scenario: argv.slice(startIdx).join(" ") }; } async function runStartCaseExperiment(scenarioInput, experimentInstruction, options = {}) { const condition = options.condition; const treatment = options.treatment; // ── Condition B/C/D: load alt prompt (B/C question-free; D deconstruct-freeform) ── let scenarioTextForPrompt = scenarioInput?.scenario; if ((condition === "B" || condition === "C" || condition === "D") && typeof scenarioTextForPrompt === "string") { const fs = require("fs"); const path = require("path"); let altPromptFile; if (condition === "B") { altPromptFile = "reconstruct-v0.5-b-noreq.md"; } else if (condition === "C") { altPromptFile = "reconstruct-v0.5-c-domain.md"; } else { // condition === "D" altPromptFile = treatment === "no21" ? "reconstruct-v0.5-d-deconstruct-no21.md" : "reconstruct-v0.5-d-deconstruct.md"; } const altPromptPath = path.join(PROJECT_ROOT, "prompts", altPromptFile); const altPromptContent = fs.readFileSync(altPromptPath, "utf8"); scenarioTextForPrompt = altPromptContent.replace("{{SCENARIO}}", scenarioTextForPrompt); } // ── Treatment validation (programmatic API) ──────────────── if (condition === "D") { const validTreatments = ["no21"]; if (treatment != null && !validTreatments.includes(treatment)) { throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21`); } } else if (treatment != null) { throw new Error(`Treatment "${treatment}" is only valid with --condition D, not --condition ${condition}`); } // ── Experiment seam bridge: supply instruction to production path ── const hadEnvVar = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION != null; const previousEnvValue = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION; if (experimentInstruction) { process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION = experimentInstruction; } let startCase = options.startCase; let reconstructionProvider = options.reconstructionProvider; let reconstructionModelName = options.reconstructionModelName; let reconstructionOnly = options.reconstructionOnly; const isMock = process.env.START_CASE_EXPERIMENT_HELPER_MOCK === "1"; // Build production request — use prepared alt prompt or raw scenario const productionRequest = { scenario: scenarioTextForPrompt ?? scenarioInput?.scenario, ...(scenarioInput?.promptVersion !== undefined ? { promptVersion: scenarioInput.promptVersion } : {}), }; // ── Condition B/C/D schema passthrough via dependencies ── let conditionSchema = null; if (condition === "B" || condition === "C") { const schemaModule = await import(PROJECT_ROOT + "/lib/reconstruction/schema-variant.js"); conditionSchema = schemaModule.conditionBJsonSchema; // question-free schema — importantUnknowns has NO question field } else if (condition === "D") { // Freeform deconstruct — explicitly no JSON structure requirement conditionSchema = null; } if (process.env.START_CASE_EXPERIMENT_PROVIDER === "openai") { if (!process.env.OPENAI_API_KEY) { throw new Error( "START_CASE_EXPERIMENT_PROVIDER=openai requires OPENAI_API_KEY", ); } const { createOpenAIReconstructionProvider } = await import( PROJECT_ROOT + "/lib/llm/provider.js" ); reconstructionProvider = createOpenAIReconstructionProvider({ apiKey: process.env.OPENAI_API_KEY, fetchImpl: fetch, }); reconstructionModelName = "gpt-5.6-terra"; reconstructionOnly = true; } // ── Condition D: freeform mode flag ──────────────── const isFreeform = condition === "D"; if (!startCase && isMock) { // Deterministic mode: skip environment checks and use inline test double. startCase = async (body, dependencies) => { // Capture the generation schema used for verification conditionSchema = dependencies.generationSchema ?? null; // Deterministic inline test double — no live model calls. const shouldFail = process.env.START_CASE_EXPERIMENT_HELPER_FORCE_FAIL === "1"; if (shouldFail) { return { success: false, error: "deterministic mock failure", statusCode: 500 }; } // In freeform mode (Condition D), the model returns raw text rather than structured data. const reconstructionResult = isFreeform ? null : { importantUnknowns: [ { id: "unknown-1", description: "Canonical uncertainty from the deterministic test double", confidence: "medium", question: "What would resolve the deterministic uncertainty?", }, ], }; // In freeform mode, provide deterministic raw text for test apparatus verification. const freeformText = isFreeform ? "Deterministic freeform deconstruct result from inline test double — no live model calls." : null; return { success: true, reconstruction: reconstructionResult, freeformText, situationGraph: { nodes: [ { id: "graph-unknown-1", label: "Canonical uncertainty", description: "Canonical uncertainty from the deterministic test double", openQuestion: "What would resolve the deterministic uncertainty?", status: "unknown", kind: "unknown", }, { id: "graph-state-1", label: "Known state", description: "A non-unknown node", status: "supported", kind: "state", }, ], edges: [], activeUnknownNodeId: null, resolvedNodeIds: [], currentSummary: "test", }, assessment: { phase: "initial", progress: 0 }, selectedQuestion: null, summary: null, diagnostics: { validationStatus: "mock", modelName: "inline-test-double", promptVersion: productionRequest.promptVersion ?? "v0.5", presentationValidationStatus: "valid", presentationValidationIssues: [], compatibilityApplied: false, compatibilityChanges: [], compatibilityWarnings: [], reconstructionOnly: dependencies.reconstructionOnly === true, graphReferenceValidation: { valid: true, errors: [] }, }, }; }; } else if (!startCase) { const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL"); const model = assertRequiredEnv("OLLAMA_MODEL"); if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") { throw new Error( `Live experiment harness refuses to use localhost fallback. ` + `OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.` ); } const { startCase: _sc } = await import(PROJECT_ROOT + "/lib/graph/orchestrator.js"); startCase = _sc; } let result; const endToEndStartedAt = Date.now(); try { const startCaseDeps = { reconstructionProvider, reconstructionModelName, reconstructionOnly, ...(isFreeform ? { freeform: true } : {}), }; if (conditionSchema) { startCaseDeps.generationSchema = conditionSchema; } result = await startCase(productionRequest, startCaseDeps); } finally { // Clean up experiment env var after execution regardless of outcome if (experimentInstruction && !hadEnvVar) { delete process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION; } else if (!hadEnvVar) { // was not set before and not set during — ensure it stays unset delete process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION; } else if (!experimentInstruction && hadEnvVar) { // restore original value that existed before if (previousEnvValue === undefined) { delete process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION; } else { process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION = previousEnvValue; } } } const output = { success: result.success, summary: result.summary ?? null, reconstruction: result.reconstruction ?? null, condition: condition ?? "A", situationGraphNodeCount: result.situationGraph?.nodes?.length ?? 0, situationGraphEdgeCount: result.situationGraph?.edges?.length ?? 0, selectedQuestion: result.selectedQuestion?.question ?? null, assessmentPhase: result.assessment?.phase ?? null, assessmentProgress: result.assessment?.progress ?? null, endToEndElapsedMs: Date.now() - endToEndStartedAt, diagnostics: result.diagnostics ?? null, providerApiPath: result.providerApiPath ?? null, error: result.success ? null : (result.error ?? "unknown"), statusCode: result.statusCode ?? (result.success ? 200 : 500), }; // ── Condition D: freeform deconstruct output — capture without production JSON schema ── if (condition === "D") { const rawText = result.freeformText ?? result.reconstruction ?? null; output.conditionType = "deconstruct"; output.freeformText = rawText; delete output.reconstruction; } if (reconstructionOnly) { if (scenarioInput?.experimentId !== undefined) { output.experimentId = scenarioInput.experimentId; } if (scenarioInput?.scenarioId !== undefined) { output.scenarioId = scenarioInput.scenarioId; } const configuredProvider = process.env.START_CASE_EXPERIMENT_PROVIDER === "openai" ? "openai" : "ollama"; const configuredModel = result.diagnostics?.modelName ?? reconstructionModelName ?? (configuredProvider === "ollama" ? process.env.OLLAMA_MODEL : undefined); const effectivePromptVersion = result.diagnostics?.promptVersion; output.execution = { condition: condition ?? "A", provider: configuredProvider, ...(configuredModel !== undefined ? { model: configuredModel } : {}), ...(effectivePromptVersion !== undefined ? { promptVersion: effectivePromptVersion } : {}), }; if (Array.isArray(result.situationGraph?.nodes)) { output.graphUnknownNodes = result.situationGraph.nodes .filter((node) => node.kind === "unknown") .map((node) => ({ id: node.id, label: node.label, description: node.description, ...(node.openQuestion !== undefined ? { openQuestion: node.openQuestion } : {}), ...(node.status !== undefined ? { status: node.status } : {}), })); } } return output; } if (require.main === module) { (async () => { // Import-only mode: prove real production seam without invoking startCase. // Used only for deterministic apparatus verification of the import path. if (process.env.START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY === "1") { try { const mod = await import(PROJECT_ROOT + "/lib/graph/orchestrator.js"); const startCaseResolved = typeof mod.startCase === "function"; const output = JSON.stringify({ success: true, mode: "import-only", startCaseResolved, }); console.log(output); process.exit(0); } catch (e) { let failureReason; if (e.code === "ERR_REQUIRE_ESM" || e.message.includes("ERR_REQUIRE_ESM")) { failureReason = "CJS cannot import ESM module"; } else if (e.code === "ERR_MODULE_NOT_FOUND" || e.code === "ERR_UNSUPPORTED_DIR_IMPORT") { failureReason = `Module resolution failed: ${e.message}`; } else { failureReason = `Import error: ${e.message}`; } const output = JSON.stringify({ success: false, mode: "import-only", startCaseResolved: false, failureReason }); console.log(output); process.exit(1); } } try { // ── Experiment seam: parse optional experiment instruction ── const experimentIdx = process.argv.indexOf("--experiment-instruction"); let experimentInstruction = null; if (experimentIdx !== -1 && experimentIdx + 1 < process.argv.length) { experimentInstruction = process.argv[experimentIdx + 1]; } // ── Condition selector: A (production) | B (question-free baseline) | C (question-free + domain-preservation) | D (deconstruct-first freeform) ── const conditionArgIdx = process.argv.indexOf("--condition"); let condition = null; if (conditionArgIdx !== -1 && conditionArgIdx + 1 < process.argv.length) { condition = process.argv[conditionArgIdx + 1]; } // ── Validate condition value ── const validConditions = ["A", "B", "C", "D"]; if (condition !== null && !validConditions.includes(condition)) { throw new Error(`Invalid condition: ${condition}. Must be one of: A, B, C, D`); } // ── Treatment selector (D only) ── const treatmentArgIdx = process.argv.indexOf("--treatment"); let treatment = null; if (treatmentArgIdx !== -1 && treatmentArgIdx + 1 < process.argv.length) { treatment = process.argv[treatmentArgIdx + 1]; } // ── Validate treatment value ── const validTreatments = ["no21"]; if (treatment !== null && !validTreatments.includes(treatment)) { throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21`); } if (treatment !== null && condition !== "D") { throw new Error(`Treatment "${treatment}" is only valid with --condition D, not --condition ${condition}`); } const scenarioInput = readScenarioInput(process.argv); const reconstructionOnly = process.argv.includes("--reconstruction-only"); const result = await runStartCaseExperiment(scenarioInput, experimentInstruction, { reconstructionOnly, condition, treatment: (condition === "D") ? treatment : undefined, }); console.log(JSON.stringify(result, null, 2)); process.exit(result.success ? 0 : 1); } catch (e) { const failure = { success: false, error: e.message ?? "unknown", statusCode: 1, }; console.log(JSON.stringify(failure)); process.exit(1); } })(); } module.exports = { runStartCaseExperiment };