confidence-engine/scripts/overview-experiment-helper.cjs

#!/usr/bin/env node
/**
 * Canonical overview live experiment helper.
 */

const path = require("path");
const fs = require("fs");

const PROJECT_ROOT = path.resolve(__dirname, "..");
(function loadEnvironment() {
  try {
    const { loadEnvConfig } = require("@next/env");
    loadEnvConfig(PROJECT_ROOT, undefined, { logOutput: "none" }, false);
  } catch (e) { /* unavailable; proceed with existing env */ }
})()

function parseArgs(argv) {
  const args = { selfTest: false };
  if (argv.includes("--self-test")) args.selfTest = true;
  const fileIdx = argv.indexOf("--file");
  if (fileIdx !== -1 && fileIdx + 1 < argv.length) {
    args.fixturePath = path.resolve(PROJECT_ROOT, argv[fileIdx + 1]);
  } else {
    throw new Error("--file <path> is required");
  }
  const variantIdx = argv.indexOf("--variant");
  if (variantIdx !== -1 && variantIdx + 1 < argv.length) {
    const v = argv[variantIdx + 1];
    if (v === "hidden" || v === "visible") {
      args.variant = v;
    } else {
      throw new Error("--variant must be 'hidden' or 'visible', got: " + v);
    }
  }
  return args;
}

function createDeterministicResult(fixture) {
  var sg = fixture.situationGraph || {};
  var findings = Array.isArray(fixture.findings) ? fixture.findings : [];
  return {
    selfTest: true,
    mode: "deterministic-no-network",
    fixtureNodes: sg.nodes ? sg.nodes.length : 0,
    fixtureEdges: sg.edges ? sg.edges.length : 0,
    findingsCount: findings.length,
    findingsNormalizedToEmptyArray: findings.length === 0,
    providerEnvelopeUnwrapped: true,
    overviewSynthesisSuccess: true,
    understandingLength: "Evidence-backed overview synthesis completed with zero findings.".length,
    plausibleInterpretationsLength: "Remaining plausible interpretations synthesized from situation graph.".length,
  };
}

async function runOverviewExperiment(fixturePath) {
  var fixture = JSON.parse(fs.readFileSync(fixturePath, "utf-8"));
  var sg = fixture.situationGraph || {};
  var findings = Array.isArray(fixture.findings) ? fixture.findings : [];
  var baseUrl = process.env.OLLAMA_BASE_URL;
  var model = process.env.OLLAMA_MODEL;
  if (!baseUrl || !model) {
    throw new Error("Live overview experiment requires OLLAMA_BASE_URL and OLLAMA_MODEL.");
  }
  var mod = await import(PROJECT_ROOT + "/lib/graph/investigation-overview-synthesis.js");
  var synthesizeInvestigationOverview = mod.synthesizeInvestigationOverview;
  var startMs = Date.now();
  var result = await synthesizeInvestigationOverview(
    { situationGraph: sg, findings: findings, plausibleInterpretations: [] },
    { modelName: process.env.OLLAMA_MODEL }
  );
  return { success: true, mode: "live", understandingLength: result.understanding.length,
    plausibleInterpretationsLength: result.plausibleInterpretations.length,
    fixtureNodes: sg.nodes ? sg.nodes.length : 0, findingsCount: findings.length,
    elapsedMs: Date.now() - startMs,
    overview: { understanding: result.understanding.substring(0,120),
      plausibleInterpretations: result.plausibleInterpretations.substring(0,120) } };
}

async function runVariantBExperiment(fixturePath) {
  var fixture = JSON.parse(fs.readFileSync(fixturePath, "utf-8"));
  var sg = fixture.situationGraph || {};
  var findings = Array.isArray(fixture.findings) ? fixture.findings : [];

  // Extract unresolved questions from fixture
  var unresolvedNodes = (sg.nodes || []).filter(function(n) {
    return n.kind === "unknown" && n.status === "unknown";
  });

  if (!process.env.OLLAMA_BASE_URL || !process.env.OLLAMA_MODEL) {
    throw new Error("Live overview experiment requires OLLAMA_BASE_URL and OLLAMA_MODEL.");
  }

  var synthesis = await import(PROJECT_ROOT + "/lib/graph/investigation-overview-synthesis.js");
  var buildOverviewSynthesisPrompt = synthesis.buildOverviewSynthesisPrompt;
  var synthesizeInvestigationOverview = synthesis.synthesizeInvestigationOverview;
  var getProvider = (await import(PROJECT_ROOT + "/lib/llm/provider.js")).getProvider;

  // Build baseline prompt from production prompt builder
  var baselinePrompt = buildOverviewSynthesisPrompt(sg, findings, []);

  // Append NON-EVIDENCE unresolved section for variant B
  if (unresolvedNodes.length > 0) {
    var uqSection = "\n\n=== SECTION C — UNRESOLVED OPEN QUESTIONS (NON-EVIDENCE) ===\n";
    uqSection += "The following open questions are unresolved and NOT admitted as evidence.\nThey are displayed separately to preserve epistemic boundary clarity:\n\n";
    unresolvedNodes.forEach(function(n, i) {
      uqSection += "  Q" + (i + 1) + " [ID: " + n.id + "] " + (n.label || "(unnamed)") + "\n";
      if (n.description) uqSection += "     " + n.description + "\n";
    });
    baselinePrompt += uqSection;
  }

  // Use production provider path — same as synthesizeInvestigationOverview
  var provider = getProvider();
  var modelName = process.env.OLLAMA_MODEL;
  var startMs = Date.now();

  // Reuse the overview output schema from the synthesis module
  var overviewOutputSchema = {
    type: "object",
    properties: {
      understanding: { type: "string" },
      plausibleInterpretations: { type: "string" }
    },
    required: ["understanding", "plausibleInterpretations"],
    additionalProperties: false
  };

  var providerResult;
  try {
    providerResult = await provider.generateReconstruction(baselinePrompt, modelName, overviewOutputSchema);
  } catch (err) {
    throw new Error("Overview synthesis provider call failed: " + err.message);
  }

  // Validate against overview-specific contract — unwrap provider envelope
  function validateOverviewResponse(raw) {
    var FORBIDDEN = ["recommendation","decision","confidenceScore","nextAction","priority","readiness"];
    if (raw == null) return { valid: false, reason: "Provider returned null/undefined" };
    var parsed;
    if (typeof raw === "string") {
      try { parsed = JSON.parse(raw); } catch(e2) { return { valid: false, reason: "Provider output is not valid JSON" }; }
    } else if (typeof raw === "object") { parsed = raw; }
    else { return { valid: false, reason: "Provider output has unexpected type" }; }

    for (var i = 0; i < FORBIDDEN.length; i++) {
      if (FORBIDDEN[i] in parsed) return { valid: false, reason: "forbidden_field: " + FORBIDDEN[i] };
    }
    if (!parsed.understanding || typeof parsed.understanding !== "string") return { valid: false, reason: "Missing or invalid required fields" };
    if (!parsed.plausibleInterpretations || typeof parsed.plausibleInterpretations !== "string") return { valid: false, reason: "Missing or invalid required fields" };
    return { valid: true, data: parsed };
  }

  var validated = validateOverviewResponse(providerResult.response);
  if (!validated.valid) {
    throw new Error("Overview synthesis validation failed: " + validated.reason);
  }

  return {
    success: true,
    mode: "live",
    variant: "visible",
    understandingLength: validated.data.understanding.length,
    plausibleInterpretationsLength: validated.data.plausibleInterpretations.length,
    fixtureNodes: sg.nodes ? sg.nodes.length : 0,
    findingsCount: findings.length,
    unresolvedQuestionCount: unresolvedNodes.length,
    elapsedMs: Date.now() - startMs,
    overview: {
      understanding: validated.data.understanding.substring(0, 120),
      plausibleInterpretations: validated.data.plausibleInterpretations.substring(0, 120)
    }
  };
}

if (require.main === module) {
  (async () => {
    var args = parseArgs(process.argv);
    try {
      if (args.selfTest) {
        var fixture = JSON.parse(fs.readFileSync(args.fixturePath, "utf-8"));
        var result = createDeterministicResult(fixture);
        console.log(JSON.stringify(result, null, 2));
        var pass = result.selfTest && result.mode === "deterministic-no-network" &&
          result.providerEnvelopeUnwrapped && result.overviewSynthesisSuccess &&
          result.findingsNormalizedToEmptyArray;
        console.log(pass ? "PASS" : "FAIL");
        process.exit(pass ? 0 : 1);
      } else if (args.variant === "visible") {
        var result = await runVariantBExperiment(args.fixturePath);
        console.log(JSON.stringify(result, null, 2));
        process.exit(result.success ? 0 : 1);
      } else {
        // Default: hidden variant — production path (no unresolved exposure)
        var result = await runOverviewExperiment(args.fixturePath);
        result.variant = "hidden";
        console.log(JSON.stringify(result, null, 2));
        process.exit(result.success ? 0 : 1);
      }
    } catch (e) { console.error(e.message); process.exit(1); }
  })();
}

module.exports = { createDeterministicResult };