test(confidence-engine): add direct initial decomposition apparatus

Establishes reusable apparatus for asking: given scenario text X, what structured initial decomposition does current production path produce? - Direct curl/Postman via existing /api/cases/start route (no new API) - Thin CJS helper at scripts/start-case-experiment-helper.cjs for Claude experiments (imports startCase directly, zero code duplication) - Zero-live-call verification: all four seam checks confirmed by existing tests (cases-start-route.test.js, start-case-summary.test.js) - No browser state, no persistence mutation, no Investigation ID required by the route itself Files: + scripts/start-case-experiment-helper.cjs (new helper script) M docs/current-handoff.md (§v0.61 apparatus documentation)
2026-09-04 13:36:25 +01:00
parent e6f2249413
commit 41ea2cb6b9
2 changed files with 206 additions and 0 deletions
@@ -0,0 +1,89 @@
+#!/usr/bin/env node
+/**
+ * Minimal startCase (initial decomposition) experiment harness.
+ *
+ * Loads .env.local, invokes startCase() through the real production path,
+ * and prints a structured result for semantic evaluation.
+ *
+ * Usage:
+ *   node scripts/start-case-experiment-helper.cjs "<scenario text>"
+ *   node scripts/start-case-experiment-helper.cjs --file scenario.json
+ *
+ * Exit codes:
+ *   0 — success (structured result printed to stdout)
+ *   1 — execution failure or invalid input
+ */
+
+const dotenv = require("dotenv");
+dotenv.config({ path: ".env.local" });
+
+function assertRequiredEnv(name) {
+  const value = process.env[name];
+  if (!value) {
+    throw new Error(
+      `Live experiment requires ${name}. Set it in .env.local.\n` +
+      `Found: OLLAMA_BASE_URL=${process.env.OLLAMA_BASE_URL ?? "(missing)"}, ` +
+      `OLLAMA_MODEL=${process.env.OLLAMA_MODEL ?? "(missing)"}`
+    );
+  }
+  return value;
+}
+
+function readScenarioInput(argv) {
+  if (argv.includes("--file")) {
+    const idx = argv.indexOf("--file");
+    if (idx + 1 >= argv.length) throw new Error("--file requires a path argument");
+    const fs = require("fs");
+    const path = argv[idx + 1];
+    return JSON.parse(fs.readFileSync(path, "utf-8"));
+  }
+  // Use all positional args as the scenario text (join with space)
+  const startIdx = argv.findIndex(a => a !== "" && !a.startsWith("-") && a !== "--");
+  if (startIdx === -1 || startIdx >= argv.length) throw new Error("Usage: node scripts/start-case-experiment-helper.cjs \"<scenario>\" [--file path.json]");
+  return { scenario: argv.slice(startIdx).join(" ") };
+}
+
+async function runStartCaseExperiment(scenarioInput) {
+  const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL");
+  const model = assertRequiredEnv("OLLAMA_MODEL");
+
+  if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
+    throw new Error(
+      `Live experiment harness refuses to use localhost fallback. ` +
+      `OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.`
+    );
+  }
+
+  const { startCase } = await import("../../lib/graph/orchestrator.js");
+  
+  const endToEndStartedAt = Date.now();
+  const result = await startCase(scenarioInput);
+  const endToEndElapsedMs = Date.now() - endToEndStartedAt;
+
+  return {
+    success: result.success,
+    summary: result.summary ?? null,
+    situationGraphNodeCount: result.situationGraph?.nodes?.length ?? 0,
+    situationGraphEdgeCount: result.situationGraph?.edges?.length ?? 0,
+    selectedQuestion: result.selectedQuestion?.question ?? null,
+    assessmentPhase: result.assessment?.phase ?? null,
+    assessmentProgress: result.assessment?.progress ?? null,
+    endToEndElapsedMs,
+    diagnostics: result.diagnostics ?? null,
+    error: result.success ? null : (result.error ?? "unknown"),
+    statusCode: result.statusCode ?? (result.success ? 200 : 500),
+  };
+}
+
+(async () => {
+  try {
+    const scenarioInput = readScenarioInput(process.argv);
+    const result = await runStartCaseExperiment(scenarioInput);
+
+    console.log(JSON.stringify(result, null, 2));
+    process.exit(result.success ? 0 : 1);
+  } catch (e) {
+    console.error(`APPARATUS FAILURE: ${e.message}`);
+    process.exit(1);
+  }
+})();