confidence-engine/scripts/start-case-experiment-helper.cjs

#!/usr/bin/env node
/**
 * Minimal startCase (initial decomposition) experiment harness.
 *
 * Loads .env.local, invokes startCase() through the real production path,
 * and prints a structured result for semantic evaluation.
 *
 * Usage:
 *   node scripts/start-case-experiment-helper.cjs "<scenario text>"
 *   node scripts/start-case-experiment-helper.cjs --file scenario.json
 *   node scripts/start-case-experiment-helper.cjs --reconstruction-only --file scenario.json
 *
 * Exit codes:
 *   0 — success (structured result printed to stdout)
 *   1 — execution failure or invalid input
 */

// Use the standard environment loader already available to this Next repository.
// loadEnvConfig mutates process.env in-place (same semantics as previous bespoke parser).
const path = require("path");
const PROJECT_ROOT = path.resolve(__dirname, "..");
(function loadEnvironment() {
  try {
    const { loadEnvConfig } = require("@next/env");
    loadEnvConfig(PROJECT_ROOT, undefined, { logOutput: "none" }, false);
  } catch {
    // loader unavailable — proceed with whatever is already set.
  }
})();

function assertRequiredEnv(name) {
  const value = process.env[name];
  if (!value) {
    throw new Error(
      `Live experiment requires ${name}. Set it in .env.local.\n` +
      `Found: OLLAMA_BASE_URL=${process.env.OLLAMA_BASE_URL ?? "(missing)"}, ` +
      `OLLAMA_MODEL=${process.env.OLLAMA_MODEL ?? "(missing)"}`
    );
  }
  return value;
}

function readScenarioInput(argv) {
  if (argv.includes("--file")) {
    const idx = argv.indexOf("--file");
    if (idx + 1 >= argv.length) throw new Error("--file requires a path argument");
    const fs = require("fs");
    const path = argv[idx + 1];
    return JSON.parse(fs.readFileSync(path, "utf-8"));
  }
  // Use all positional args as the scenario text (join with space)
  const startIdx = argv.findIndex(a => a !== "" && !a.startsWith("-") && a !== "--");
  if (startIdx === -1 || startIdx >= argv.length) throw new Error("Usage: node scripts/start-case-experiment-helper.cjs \"<scenario>\" [--file path.json]");
  return { scenario: argv.slice(startIdx).join(" ") };
}

async function runStartCaseExperiment(scenarioInput, experimentInstruction, options = {}) {
  // ── Experiment seam bridge: supply instruction to production path ──
  const hadEnvVar = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION != null;
  const previousEnvValue = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION;
  if (experimentInstruction) {
    process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION = experimentInstruction;
  }

  let startCase = options.startCase;
  let reconstructionProvider = options.reconstructionProvider;
  let reconstructionModelName = options.reconstructionModelName;
  let reconstructionOnly = options.reconstructionOnly;
  const isMock = process.env.START_CASE_EXPERIMENT_HELPER_MOCK === "1";

  if (process.env.START_CASE_EXPERIMENT_PROVIDER === "openai") {
    if (!process.env.OPENAI_API_KEY) {
      throw new Error(
        "START_CASE_EXPERIMENT_PROVIDER=openai requires OPENAI_API_KEY",
      );
    }
    const { createOpenAIReconstructionProvider } = await import(
      PROJECT_ROOT + "/lib/llm/provider.js"
    );
    reconstructionProvider = createOpenAIReconstructionProvider({
      apiKey: process.env.OPENAI_API_KEY,
      fetchImpl: fetch,
    });
    reconstructionModelName = "gpt-5.6-terra";
    reconstructionOnly = true;
  }

  if (!startCase && isMock) {
    // Deterministic mode: skip environment checks and use inline test double.
    startCase = async (_body, dependencies) => {
      // Deterministic inline test double — no live model calls.
      const shouldFail = process.env.START_CASE_EXPERIMENT_HELPER_FORCE_FAIL === "1";
      if (shouldFail) {
        return { success: false, error: "deterministic mock failure", statusCode: 500 };
      }
      return {
        success: true,
        situationGraph: { nodes: [], edges: [], activeUnknownNodeId: null, resolvedNodeIds: [], currentSummary: "test" },
        assessment: { phase: "initial", progress: 0 },
        selectedQuestion: null,
        summary: null,
        diagnostics: {
          validationStatus: "mock",
          modelName: "inline-test-double",
          reconstructionOnly: dependencies.reconstructionOnly === true,
          graphReferenceValidation: { valid: true, errors: [] },
        },
      };
    };
  } else if (!startCase) {
    const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL");
    const model = assertRequiredEnv("OLLAMA_MODEL");

    if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
      throw new Error(
        `Live experiment harness refuses to use localhost fallback. ` +
        `OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.`
      );
    }

    const { startCase: _sc } = await import(PROJECT_ROOT + "/lib/graph/orchestrator.js");
    startCase = _sc;
  }

  let result;
  const endToEndStartedAt = Date.now();
  try {
    result = await startCase(scenarioInput, {
      reconstructionProvider,
      reconstructionModelName,
      reconstructionOnly,
    });
  } finally {
    // Clean up experiment env var after execution regardless of outcome
    if (experimentInstruction && !hadEnvVar) {
      delete process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION;
    } else if (!hadEnvVar) {
      // was not set before and not set during — ensure it stays unset
      delete process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION;
    } else if (!experimentInstruction && hadEnvVar) {
      // restore original value that existed before
      if (previousEnvValue === undefined) {
        delete process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION;
      } else {
        process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION = previousEnvValue;
      }
    }
  }

  return {
    success: result.success,
    summary: result.summary ?? null,
    reconstruction: result.reconstruction ?? null,
    situationGraphNodeCount: result.situationGraph?.nodes?.length ?? 0,
    situationGraphEdgeCount: result.situationGraph?.edges?.length ?? 0,
    selectedQuestion: result.selectedQuestion?.question ?? null,
    assessmentPhase: result.assessment?.phase ?? null,
    assessmentProgress: result.assessment?.progress ?? null,
    endToEndElapsedMs: Date.now() - endToEndStartedAt,
    diagnostics: result.diagnostics ?? null,
    providerApiPath: result.providerApiPath ?? null,
    error: result.success ? null : (result.error ?? "unknown"),
    statusCode: result.statusCode ?? (result.success ? 200 : 500),
  };
}

if (require.main === module) {
  (async () => {
  // Import-only mode: prove real production seam without invoking startCase.
  // Used only for deterministic apparatus verification of the import path.
  if (process.env.START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY === "1") {
    try {
      const mod = await import(PROJECT_ROOT + "/lib/graph/orchestrator.js");
      const startCaseResolved = typeof mod.startCase === "function";
      const output = JSON.stringify({
        success: true,
        mode: "import-only",
        startCaseResolved,
      });
      console.log(output);
      process.exit(0);
    } catch (e) {
      let failureReason;
      if (e.code === "ERR_REQUIRE_ESM" || e.message.includes("ERR_REQUIRE_ESM")) {
        failureReason = "CJS cannot import ESM module";
      } else if (e.code === "ERR_MODULE_NOT_FOUND" || e.code === "ERR_UNSUPPORTED_DIR_IMPORT") {
        failureReason = `Module resolution failed: ${e.message}`;
      } else {
        failureReason = `Import error: ${e.message}`;
      }
      const output = JSON.stringify({ success: false, mode: "import-only", startCaseResolved: false, failureReason });
      console.log(output);
      process.exit(1);
    }
  }

  try {
    // ── Experiment seam: parse optional experiment instruction ──
    const experimentIdx = process.argv.indexOf("--experiment-instruction");
    let experimentInstruction = null;
    if (experimentIdx !== -1 && experimentIdx + 1 < process.argv.length) {
      experimentInstruction = process.argv[experimentIdx + 1];
    }

    const scenarioInput = readScenarioInput(process.argv);
    const reconstructionOnly = process.argv.includes("--reconstruction-only");
    const result = await runStartCaseExperiment(scenarioInput, experimentInstruction, {
      reconstructionOnly,
    });

    console.log(JSON.stringify(result, null, 2));
    process.exit(result.success ? 0 : 1);
  } catch (e) {
    const failure = {
      success: false,
      error: e.message ?? "unknown",
      statusCode: 1,
    };
    console.log(JSON.stringify(failure));
    process.exit(1);
  }
  })();
}

module.exports = { runStartCaseExperiment };