confidence-engine/scripts/start-case-experiment-helper.cjs

#!/usr/bin/env node
/**
 * Minimal startCase (initial decomposition) experiment harness.
 *
 * Loads .env.local, invokes startCase() through the real production path,
 * and prints a structured result for semantic evaluation.
 *
 * Usage:
 *   node scripts/start-case-experiment-helper.cjs "<scenario text>"
 *   node scripts/start-case-experiment-helper.cjs --file scenario.json
 *   node scripts/start-case-experiment-helper.cjs --reconstruction-only --file scenario.json
 *
 * Exit codes:
 *   0 — success (structured result printed to stdout)
 *   1 — execution failure or invalid input
 */

// Use the standard environment loader already available to this Next repository.
// loadEnvConfig mutates process.env in-place (same semantics as previous bespoke parser).
const path = require("path");
const PROJECT_ROOT = path.resolve(__dirname, "..");
(function loadEnvironment() {
  try {
    const { loadEnvConfig } = require("@next/env");
    loadEnvConfig(PROJECT_ROOT, undefined, { logOutput: "none" }, false);
  } catch {
    // loader unavailable — proceed with whatever is already set.
  }
})();

function assertRequiredEnv(name) {
  const value = process.env[name];
  if (!value) {
    throw new Error(
      `Live experiment requires ${name}. Set it in .env.local.\n` +
      `Found: OLLAMA_BASE_URL=${process.env.OLLAMA_BASE_URL ?? "(missing)"}, ` +
      `OLLAMA_MODEL=${process.env.OLLAMA_MODEL ?? "(missing)"}`
    );
  }
  return value;
}

function readScenarioInput(argv) {
  if (argv.includes("--file")) {
    const idx = argv.indexOf("--file");
    if (idx + 1 >= argv.length) throw new Error("--file requires a path argument");
    const fs = require("fs");
    const path = argv[idx + 1];
    return JSON.parse(fs.readFileSync(path, "utf-8"));
  }
  // Use all positional args as the scenario text (join with space)
  const startIdx = argv.findIndex(a => a !== "" && !a.startsWith("-") && a !== "--");
  if (startIdx === -1 || startIdx >= argv.length) throw new Error("Usage: node scripts/start-case-experiment-helper.cjs \"<scenario>\" [--file path.json]");
  return { scenario: argv.slice(startIdx).join(" ") };
}

async function runStartCaseExperiment(scenarioInput, experimentInstruction, options = {}) {
  const condition = options.condition;
  const treatment = options.treatment;

  // ── Condition B/C/D/E: load alt prompt (B/C question-free; D deconstruct-freeform; E investigability-first freeform) ──
  let scenarioTextForPrompt = scenarioInput?.scenario;
  if ((condition === "B" || condition === "C" || condition === "D" || condition === "E") && typeof scenarioTextForPrompt === "string") {
    const fs = require("fs");
    const path = require("path");
    let altPromptFile;
    if (condition === "B") {
      altPromptFile = "reconstruct-v0.5-b-noreq.md";
    } else if (condition === "C") {
      altPromptFile = "reconstruct-v0.5-c-domain.md";
    } else if (condition === "D") {
      altPromptFile = treatment === "no21"
        ? "reconstruct-v0.5-d-deconstruct-no21.md"
        : "reconstruct-v0.5-d-deconstruct.md";
    } else {
      // condition === "E"
      altPromptFile = "reconstruct-v0.5-e-investigability-first.md";
    }
    const altPromptPath = path.join(PROJECT_ROOT, "prompts", altPromptFile);
    const altPromptContent = fs.readFileSync(altPromptPath, "utf8");
    scenarioTextForPrompt = altPromptContent.replace("{{SCENARIO}}", scenarioTextForPrompt);
  }

  // ── Treatment validation (programmatic API) ────────────────
  if (condition === "D") {
    const validTreatments = ["no21"];
    if (treatment != null && !validTreatments.includes(treatment)) {
      throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21`);
    }
  } else if (condition === "E") {
    const validTreatments = ["e-investigability-first"];
    if (treatment != null && !validTreatments.includes(treatment)) {
      throw new Error(`Invalid treatment: ${treatment}. Must be one of: e-investigability-first`);
    }
  } else if (treatment != null) {
    throw new Error(`Treatment "${treatment}" is only valid with --condition D or E, not --condition ${condition}`);
  }

  // ── Experiment seam bridge: supply instruction to production path ──
  const hadEnvVar = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION != null;
  const previousEnvValue = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION;
  if (experimentInstruction) {
    process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION = experimentInstruction;
  }

  let startCase = options.startCase;
  let reconstructionProvider = options.reconstructionProvider;
  let reconstructionModelName = options.reconstructionModelName;
  let reconstructionOnly = options.reconstructionOnly;
  const isMock = process.env.START_CASE_EXPERIMENT_HELPER_MOCK === "1";

  // Build production request — use prepared alt prompt or raw scenario
  const productionRequest = {
    scenario: scenarioTextForPrompt ?? scenarioInput?.scenario,
    ...(scenarioInput?.promptVersion !== undefined
      ? { promptVersion: scenarioInput.promptVersion }
      : {}),
  };

  // ── Condition B/C/D/E schema passthrough via dependencies ──
  let conditionSchema = null;
  if (condition === "B" || condition === "C") {
    const schemaModule = await import(PROJECT_ROOT + "/lib/reconstruction/schema-variant.js");
    conditionSchema = schemaModule.conditionBJsonSchema; // question-free schema — importantUnknowns has NO question field
  } else if (condition === "D" || condition === "E") {
    // Freeform deconstruct / investigability-first — explicitly no JSON structure requirement
    conditionSchema = null;
  }

  if (process.env.START_CASE_EXPERIMENT_PROVIDER === "openai") {
    if (!process.env.OPENAI_API_KEY) {
      throw new Error(
        "START_CASE_EXPERIMENT_PROVIDER=openai requires OPENAI_API_KEY",
      );
    }
    const { createOpenAIReconstructionProvider } = await import(
      PROJECT_ROOT + "/lib/llm/provider.js"
    );
    reconstructionProvider = createOpenAIReconstructionProvider({
      apiKey: process.env.OPENAI_API_KEY,
      fetchImpl: fetch,
    });
    reconstructionModelName = "gpt-5.6-terra";
    reconstructionOnly = true;
  }

  // ── Condition D/E: freeform mode flag ────────────────
  const isFreeform = condition === "D" || condition === "E";

  if (!startCase && isMock) {
    // Deterministic mode: skip environment checks and use inline test double.
    startCase = async (body, dependencies) => {
      // Capture the generation schema used for verification
      conditionSchema = dependencies.generationSchema ?? null;

      // Deterministic inline test double — no live model calls.
      const shouldFail = process.env.START_CASE_EXPERIMENT_HELPER_FORCE_FAIL === "1";
      if (shouldFail) {
        return { success: false, error: "deterministic mock failure", statusCode: 500 };
      }

      // In freeform mode (Condition D), the model returns raw text rather than structured data.
      const reconstructionResult = isFreeform
        ? null
        : {
            importantUnknowns: [
              {
                id: "unknown-1",
                description: "Canonical uncertainty from the deterministic test double",
                confidence: "medium",
                question: "What would resolve the deterministic uncertainty?",
              },
            ],
          };

      // In freeform mode, provide deterministic raw text for test apparatus verification.
      const freeformText = isFreeform
        ? "Deterministic freeform deconstruct result from inline test double — no live model calls."
        : null;

      return {
        success: true,
        reconstruction: reconstructionResult,
        freeformText,
        situationGraph: {
          nodes: [
            {
              id: "graph-unknown-1",
              label: "Canonical uncertainty",
              description: "Canonical uncertainty from the deterministic test double",
              openQuestion: "What would resolve the deterministic uncertainty?",
              status: "unknown",
              kind: "unknown",
            },
            {
              id: "graph-state-1",
              label: "Known state",
              description: "A non-unknown node",
              status: "supported",
              kind: "state",
            },
          ],
          edges: [],
          activeUnknownNodeId: null,
          resolvedNodeIds: [],
          currentSummary: "test",
        },
        assessment: { phase: "initial", progress: 0 },
        selectedQuestion: null,
        summary: null,
        diagnostics: {
          validationStatus: "mock",
          modelName: "inline-test-double",
          promptVersion: productionRequest.promptVersion ?? "v0.5",
          presentationValidationStatus: "valid",
          presentationValidationIssues: [],
          compatibilityApplied: false,
          compatibilityChanges: [],
          compatibilityWarnings: [],
          reconstructionOnly: dependencies.reconstructionOnly === true,
          graphReferenceValidation: { valid: true, errors: [] },
        },
      };
    };
  } else if (!startCase) {
    const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL");
    const model = assertRequiredEnv("OLLAMA_MODEL");

    if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
      throw new Error(
        `Live experiment harness refuses to use localhost fallback. ` +
        `OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.`
      );
    }

    const { startCase: _sc } = await import(PROJECT_ROOT + "/lib/graph/orchestrator.js");
    startCase = _sc;
  }

  let result;
  const endToEndStartedAt = Date.now();
  try {
    const startCaseDeps = {
      reconstructionProvider,
      reconstructionModelName,
      reconstructionOnly,
      ...(isFreeform ? { freeform: true } : {}),
    };
    if (conditionSchema) {
      startCaseDeps.generationSchema = conditionSchema;
    }
    result = await startCase(productionRequest, startCaseDeps);
  } finally {
    // Clean up experiment env var after execution regardless of outcome
    if (experimentInstruction && !hadEnvVar) {
      delete process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION;
    } else if (!hadEnvVar) {
      // was not set before and not set during — ensure it stays unset
      delete process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION;
    } else if (!experimentInstruction && hadEnvVar) {
      // restore original value that existed before
      if (previousEnvValue === undefined) {
        delete process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION;
      } else {
        process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION = previousEnvValue;
      }
    }
  }

  const output = {
    success: result.success,
    summary: result.summary ?? null,
    reconstruction: result.reconstruction ?? null,
    condition: condition ?? "A",
    situationGraphNodeCount: result.situationGraph?.nodes?.length ?? 0,
    situationGraphEdgeCount: result.situationGraph?.edges?.length ?? 0,
    selectedQuestion: result.selectedQuestion?.question ?? null,
    assessmentPhase: result.assessment?.phase ?? null,
    assessmentProgress: result.assessment?.progress ?? null,
    endToEndElapsedMs: Date.now() - endToEndStartedAt,
    diagnostics: result.diagnostics ?? null,
    providerApiPath: result.providerApiPath ?? null,
    error: result.success ? null : (result.error ?? "unknown"),
    statusCode: result.statusCode ?? (result.success ? 200 : 500),
  };

  // ── Condition D/E: freeform deconstruct / investigability-first output — capture without production JSON schema ──
  if (condition === "D" || condition === "E") {
    const rawText = result.freeformText ?? result.reconstruction ?? null;
    output.conditionType = condition === "D" ? "deconstruct" : "investigability-first";
    output.freeformText = rawText;
    delete output.reconstruction;
  }

  if (reconstructionOnly) {
    if (scenarioInput?.experimentId !== undefined) {
      output.experimentId = scenarioInput.experimentId;
    }
    if (scenarioInput?.scenarioId !== undefined) {
      output.scenarioId = scenarioInput.scenarioId;
    }

    const configuredProvider =
      process.env.START_CASE_EXPERIMENT_PROVIDER === "openai" ? "openai" : "ollama";
    const configuredModel =
      result.diagnostics?.modelName ??
      reconstructionModelName ??
      (configuredProvider === "ollama" ? process.env.OLLAMA_MODEL : undefined);
    const effectivePromptVersion = result.diagnostics?.promptVersion;

    output.execution = {
      condition: condition ?? "A",
      provider: configuredProvider,
      ...(configuredModel !== undefined ? { model: configuredModel } : {}),
      ...(effectivePromptVersion !== undefined
        ? { promptVersion: effectivePromptVersion }
        : {}),
    };

    if (Array.isArray(result.situationGraph?.nodes)) {
      output.graphUnknownNodes = result.situationGraph.nodes
        .filter((node) => node.kind === "unknown")
        .map((node) => ({
          id: node.id,
          label: node.label,
          description: node.description,
          ...(node.openQuestion !== undefined
            ? { openQuestion: node.openQuestion }
            : {}),
          ...(node.status !== undefined ? { status: node.status } : {}),
        }));
    }
  }

  return output;
}

if (require.main === module) {
  (async () => {
  // Import-only mode: prove real production seam without invoking startCase.
  // Used only for deterministic apparatus verification of the import path.
  if (process.env.START_CASE_EXPERIMENT_HELPER_IMPORT_ONLY === "1") {
    try {
      const mod = await import(PROJECT_ROOT + "/lib/graph/orchestrator.js");
      const startCaseResolved = typeof mod.startCase === "function";
      const output = JSON.stringify({
        success: true,
        mode: "import-only",
        startCaseResolved,
      });
      console.log(output);
      process.exit(0);
    } catch (e) {
      let failureReason;
      if (e.code === "ERR_REQUIRE_ESM" || e.message.includes("ERR_REQUIRE_ESM")) {
        failureReason = "CJS cannot import ESM module";
      } else if (e.code === "ERR_MODULE_NOT_FOUND" || e.code === "ERR_UNSUPPORTED_DIR_IMPORT") {
        failureReason = `Module resolution failed: ${e.message}`;
      } else {
        failureReason = `Import error: ${e.message}`;
      }
      const output = JSON.stringify({ success: false, mode: "import-only", startCaseResolved: false, failureReason });
      console.log(output);
      process.exit(1);
    }
  }

  try {
    // ── Experiment seam: parse optional experiment instruction ──
    const experimentIdx = process.argv.indexOf("--experiment-instruction");
    let experimentInstruction = null;
    if (experimentIdx !== -1 && experimentIdx + 1 < process.argv.length) {
      experimentInstruction = process.argv[experimentIdx + 1];
    }

    // ── Condition selector: A (production) | B (question-free baseline) | C (question-free + domain-preservation) | D (deconstruct-first freeform) ──
    const conditionArgIdx = process.argv.indexOf("--condition");
    let condition = null;
    if (conditionArgIdx !== -1 && conditionArgIdx + 1 < process.argv.length) {
      condition = process.argv[conditionArgIdx + 1];
    }

    // ── Validate condition value ──
    const validConditions = ["A", "B", "C", "D", "E"];
    if (condition !== null && !validConditions.includes(condition)) {
      throw new Error(`Invalid condition: ${condition}. Must be one of: A, B, C, D, E`);
    }

    // ── Treatment selector (D and E only) ──
    const treatmentArgIdx = process.argv.indexOf("--treatment");
    let treatment = null;
    if (treatmentArgIdx !== -1 && treatmentArgIdx + 1 < process.argv.length) {
      treatment = process.argv[treatmentArgIdx + 1];
    }

    // ── Validate treatment value ──
    const validDTreatments = ["no21"];
    const validETreatments = ["e-investigability-first"];
    if (treatment !== null && condition === "D" && !validDTreatments.includes(treatment)) {
      throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21`);
    }
    if (treatment !== null && condition === "E" && !validETreatments.includes(treatment)) {
      throw new Error(`Invalid treatment: ${treatment}. Must be one of: e-investigability-first`);
    }
    if (treatment !== null && !["D", "E"].includes(condition)) {
      throw new Error(`Treatment "${treatment}" is only valid with --condition D or E, not --condition ${condition}`);
    }

    const scenarioInput = readScenarioInput(process.argv);
    const reconstructionOnly = process.argv.includes("--reconstruction-only");
    const result = await runStartCaseExperiment(scenarioInput, experimentInstruction, {
      reconstructionOnly,
      condition,
      treatment: (condition === "D" || condition === "E") ? treatment : undefined,
    });

    console.log(JSON.stringify(result, null, 2));
    process.exit(result.success ? 0 : 1);
  } catch (e) {
    const failure = {
      success: false,
      error: e.message ?? "unknown",
      statusCode: 1,
    };
    console.log(JSON.stringify(failure));
    process.exit(1);
  }
  })();
}

module.exports = { runStartCaseExperiment };