confidence-engine/scripts/experimental/rto-focused-vs-global-comparison.mjs

import fs from "fs/promises";
import path from "path";
import dotenv from "dotenv";

import fixture from "../../tests/fixtures/live-product-launch-update-response.json" with { type: "json" };
import { buildGraphUpdatePrompt } from "../../lib/graph/prompt-builder.js";
import { assertConfig } from "../../lib/config.js";

import { createRequire } from "module";
const require = createRequire(import.meta.url);
const { runLiveExperiment } = require("../../tests/graph/live-update-experiment-helper.cjs");

// Non-production diagnostics: an independent request captures raw-response
// text and Ollama completion metadata (done, done_reason, prompt_eval_count,
// eval_count) before delegating JSON recovery to the inline helper.
import { runFocusedDiagnostic } from "./focus-diagnostics-helper.mjs";

dotenv.config({ path: ".env.local" });

const TARGET_NODE_ID = "nxmeiab";
const FIXED_QUESTION =
  "What evidence would clarify whether competitors are actively developing similar products and how soon they might release them?";
const FIXED_ANSWER =
  "Two competitors have publicly announced products aimed at the same customer problem. One says it expects a beta within six months, while the other has not announced a release date. We do not yet know how closely either product matches ours.";

function getGraph() {
  return structuredClone(fixture.updatedSituationGraph);
}

function getTargetNode(graph) {
  const targetNode = graph.nodes.find((node) => node.id === TARGET_NODE_ID);
  if (!targetNode || targetNode.kind !== "unknown" || targetNode.status === "resolved") {
    throw new Error("Fixed target node nxmeiab is not present as unresolved unknown in fixture.");
  }
  return targetNode;
}

function buildFocusedPrompt({ targetNode, centralStatement, question, answer }) {
  return `You are performing focused answer deconstruction for one explicitly user-chosen investigation.

Return exactly one JSON object. Return JSON only.

This is NOT a graph update task.
Do NOT output graph mutations.
Do NOT output selection, ranking, ownership, recommendation, confidence, or next-best-question semantics.
Do NOT include any of these fields: addedNodes, updatedNodes, removedNodes, addedEdges, removedEdges, resolvedNodeIds, activeUnknownNodeId, selectedQuestion.

Required top-level fields:
- targetNodeId
- observations
- uncertainties
- assumptions
- relationships
- possibleFollowUpQuestions

Field rules:
- targetNodeId must be exactly "${TARGET_NODE_ID}"
- observations: only statements directly supported by the answer
- uncertainties: only things the answer explicitly leaves unknown or unclear
- assumptions: include only if the answer itself relies on an assumption
- relationships: only direct supported relationships among extracted items, each with { from, to, type, rationale }
- possibleFollowUpQuestions: unresolved questions genuinely exposed by this answer, unranked

Focused case context:
- target label: ${targetNode.label}
- target description: ${targetNode.description}
- central case statement: ${centralStatement}

Question:
${question}

Answer:
${answer}`;
}

function assertRequiredEnv(name) {
  const value = process.env[name];
  if (!value) {
    throw new Error(`${name} is required in .env.local`);
  }
  return value;
}

function validateEnvironment() {
  const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL");
  if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
    throw new Error(`Comparison apparatus refuses localhost fallback. OLLAMA_BASE_URL=${baseUrl}`);
  }
  const config = assertConfig();
  return { baseUrl, modelName: config.OLLAMA_MODEL };
}

function buildFocusedPlan() {
  const graph = getGraph();
  const targetNode = getTargetNode(graph);
  const prompt = buildFocusedPrompt({
    targetNode,
    centralStatement: graph.centralStatement,
    question: FIXED_QUESTION,
    answer: FIXED_ANSWER,
  });

  return {
    path: "focused",
    targetNodeId: TARGET_NODE_ID,
    modelTimingBoundary:
      "Immediately before provider.generateReconstruction(prompt, modelName) to immediately after it resolves.",
    inputCharacterCount: prompt.length,
    prompt,
    focusedContextSummary: [
      "target node id and label",
      "target node description",
      "central case statement",
      "fixed question",
      "fixed answer",
    ],
  };
}

function buildGlobalPlan() {
  const graph = getGraph();
  const prompt = buildGraphUpdatePrompt({
    situationGraph: graph,
    previousQuestion: FIXED_QUESTION,
    answer: FIXED_ANSWER,
  });

  return {
    path: "global",
    targetNodeId: TARGET_NODE_ID,
    modelTimingBoundary:
      "Immediately before updateCase() calls provider.generateReconstruction(prompt, modelName) to immediately after it resolves inside orchestrator.",
    endToEndTimingBoundary:
      "Immediately before updateCase() entry to immediately after full updateCase() result returns.",
    inputCharacterCount: prompt.length,
    prompt,
    graph,
  };
}

async function runFocused() {
  const { baseUrl, modelName } = validateEnvironment();
  const plan = buildFocusedPlan();

  // ---- non-production diagnostics wrapper ----
  // Independent request captures raw response + completion metadata.
  // The diagnostics helper mirrors the production JSON-recovery logic inline
  // (since recoverJson is not exported) and delegates parse back to that
  // implementation. The comparison runner only observes result/error objects.
  const diagResult = await runFocusedDiagnostic({ baseUrl, modelName, prompt: plan.prompt });

  try {
    if (diagResult.error) {
      throw diagResult.error;
    }
    // Successful parse via production helper — proceed to standard artifact
    const raw = diagResult.parsedResult;

    const artifact = {
      path: "focused",
      targetNodeId: TARGET_NODE_ID,
      modelName,
      modelElapsedMs: diagResult.diagnostics.modelElapsedMs,
      inputCharacterCount: plan.inputCharacterCount,
      structuredResult: raw,
    };

    const artifactPath = path.resolve(
      "tests/experimental/artifacts/rto-focused-vs-global-focused.json",
    );
    await fs.mkdir(path.dirname(artifactPath), { recursive: true });
    await fs.writeFile(artifactPath, JSON.stringify(artifact, null, 2));
    console.log(JSON.stringify({ artifactPath, modelElapsedMs: diagResult.diagnostics.modelElapsedMs, inputCharacterCount: plan.inputCharacterCount }, null, 2));
    return;
  } catch (err) {
    const focusedDiagnostics = err.focusedDiagnostics ?? null;

    if (!focusedDiagnostics) {
      // Should not happen — every DiagnosticError carries diagnostics.
      console.error("Unexpected: no diagnostics captured before parse failure.");
      process.exit(1);
    }

    const errorArtifact = {
      path: "focused",
      modelName: focusedDiagnostics.apiUsed === "/api/chat" ? modelName : modelName,
      inputCharacterCount: plan.inputCharacterCount,
      rawResponseText: focusedDiagnostics.rawResponseText?.slice(0, 4096),
      rawResponseCharacterCount: focusedDiagnostics.rawResponseCharacterCount,
      done: focusedDiagnostics.done,
      doneReason: focusedDiagnostics.doneReason,
      promptEvalCount: focusedDiagnostics.promptEvalCount,
      evalCount: focusedDiagnostics.evalCount,
      errorType: err.name,
      errorMessage: err.message.slice(0, 4096),
    };

    const artifactPath = path.resolve(
      "tests/experimental/artifacts/rto-focused-vs-global-focused-failure.json",
    );
    await fs.mkdir(path.dirname(artifactPath), { recursive: true });
    await fs.writeFile(artifactPath, JSON.stringify(errorArtifact, null, 2));

    console.log(JSON.stringify({
      artifactPath,
      errorType: err.name,
      rawResponseCharacterCount: focusedDiagnostics.rawResponseCharacterCount,
      done: focusedDiagnostics.done,
      doneReason: focusedDiagnostics.doneReason,
    }, null, 2));
  }
}

async function runGlobal() {
  const { modelName } = validateEnvironment();
  const plan = buildGlobalPlan();

  const startedAt = Date.now();
  const result = await runLiveExperiment({
    graph: plan.graph,
    previousQuestion: FIXED_QUESTION,
    answer: FIXED_ANSWER,
  });
  const endToEndElapsedMs = Date.now() - startedAt;

  const artifact = {
    path: "global",
    targetNodeId: TARGET_NODE_ID,
    modelName,
    modelElapsedMs: result.modelElapsedMs,
    providerGenerateCalls: result.providerGenerateCalls ?? 0,
    providerTimingCaptured: typeof result.modelElapsedMs === "number",
    endToEndElapsedMs: result.endToEndElapsedMs ?? endToEndElapsedMs,
    inputCharacterCount: plan.inputCharacterCount,
    graphUpdateResultSummary: {
      userSupportedMeaning: result.userSupportedMeaning,
      possibleInference: result.possibleInference,
      proposalValidation: result.proposalValidation,
      selectedQuestion: result.selectedQuestion,
    },
    answerDerivedComparisonExtraction: {
      observations: result.userSupportedMeaning ? [result.userSupportedMeaning] : "not directly exposed",
      uncertainties: result.possibleInference ? [result.possibleInference] : "not directly exposed",
      assumptions: "not directly exposed",
      relationships: "not directly exposed",
      newlySurfacedUnknownsOrQuestions: result.selectedQuestion ?? "not directly exposed",
    },
  };

  const artifactPath = path.resolve(
    "tests/experimental/artifacts/rto-focused-vs-global-global.json",
  );
  await fs.mkdir(path.dirname(artifactPath), { recursive: true });
  await fs.writeFile(artifactPath, JSON.stringify(artifact, null, 2));
  console.log(JSON.stringify({ artifactPath, modelElapsedMs: result.modelElapsedMs, endToEndElapsedMs: result.endToEndElapsedMs ?? endToEndElapsedMs, inputCharacterCount: plan.inputCharacterCount }, null, 2));
}

function printStaticValidation() {
  const { modelName } = validateEnvironment();
  const focused = buildFocusedPlan();
  const global = buildGlobalPlan();

  const summary = {
    modelName,
    fixedTargetNodeId: TARGET_NODE_ID,
    fixedQuestion: FIXED_QUESTION,
    fixedAnswer: FIXED_ANSWER,
    focused: {
      resolvesTo: "scripts/experimental/rto-focused-vs-global-comparison-equivalent focused provider path",
      wholeGraphSupplied: false,
      inputCharacterCount: focused.inputCharacterCount,
      modelTimingBoundary: focused.modelTimingBoundary,
    },
    global: {
      resolvesTo: "tests/graph/live-update-experiment-helper.cjs -> updateCase() production path",
      wholeGraphSupplied: true,
      inputCharacterCount: global.inputCharacterCount,
      modelTimingBoundary: global.modelTimingBoundary,
      endToEndTimingBoundary: global.endToEndTimingBoundary,
      providerTimingSource: "non-production timed provider wrapper injected through updateCase dependencies.provider",
    },
    commands: {
      focused: "node scripts/experimental/rto-focused-vs-global-comparison.mjs --focused",
      global: "node scripts/experimental/rto-focused-vs-global-comparison.mjs --global",
    },
  };

  console.log(JSON.stringify(summary, null, 2));
}

const mode = process.argv[2] ?? "--validate";

if (mode === "--focused") {
  runFocused().catch((error) => {
    console.error(error instanceof Error ? error.message : String(error));
    process.exit(1);
  });
} else if (mode === "--global") {
  runGlobal().catch((error) => {
    console.error(error instanceof Error ? error.message : String(error));
    process.exit(1);
  });
} else if (mode === "--validate") {
  printStaticValidation();
} else {
  console.error(`Unknown mode: ${mode}`);
  process.exit(1);
}