test(experiment): checkpoint focused vs global comparison apparatus

2026-08-18 18:34:08 +01:00
parent 2f6c90b027
commit 952cb442b5
2 changed files with 277 additions and 1 deletions
@@ -0,0 +1,255 @@
+import fs from "fs/promises";
+import path from "path";
+import dotenv from "dotenv";
+
+import fixture from "../../tests/fixtures/live-product-launch-update-response.json" with { type: "json" };
+import { buildGraphUpdatePrompt } from "../../lib/graph/prompt-builder.js";
+import { getProvider } from "../../lib/llm/provider.js";
+import { assertConfig } from "../../lib/config.js";
+
+import { createRequire } from "module";
+const require = createRequire(import.meta.url);
+const { runLiveExperiment } = require("../../tests/graph/live-update-experiment-helper.cjs");
+
+dotenv.config({ path: ".env.local" });
+
+const TARGET_NODE_ID = "nxmeiab";
+const FIXED_QUESTION =
+  "What evidence would clarify whether competitors are actively developing similar products and how soon they might release them?";
+const FIXED_ANSWER =
+  "Two competitors have publicly announced products aimed at the same customer problem. One says it expects a beta within six months, while the other has not announced a release date. We do not yet know how closely either product matches ours.";
+
+function getGraph() {
+  return structuredClone(fixture.updatedSituationGraph);
+}
+
+function getTargetNode(graph) {
+  const targetNode = graph.nodes.find((node) => node.id === TARGET_NODE_ID);
+  if (!targetNode || targetNode.kind !== "unknown" || targetNode.status === "resolved") {
+    throw new Error("Fixed target node nxmeiab is not present as unresolved unknown in fixture.");
+  }
+  return targetNode;
+}
+
+function buildFocusedPrompt({ targetNode, centralStatement, question, answer }) {
+  return `You are performing focused answer deconstruction for one explicitly user-chosen investigation.
+
+Return exactly one JSON object. Return JSON only.
+
+This is NOT a graph update task.
+Do NOT output graph mutations.
+Do NOT output selection, ranking, ownership, recommendation, confidence, or next-best-question semantics.
+Do NOT include any of these fields: addedNodes, updatedNodes, removedNodes, addedEdges, removedEdges, resolvedNodeIds, activeUnknownNodeId, selectedQuestion.
+
+Required top-level fields:
+- targetNodeId
+- observations
+- uncertainties
+- assumptions
+- relationships
+- possibleFollowUpQuestions
+
+Field rules:
+- targetNodeId must be exactly "${TARGET_NODE_ID}"
+- observations: only statements directly supported by the answer
+- uncertainties: only things the answer explicitly leaves unknown or unclear
+- assumptions: include only if the answer itself relies on an assumption
+- relationships: only direct supported relationships among extracted items, each with { from, to, type, rationale }
+- possibleFollowUpQuestions: unresolved questions genuinely exposed by this answer, unranked
+
+Focused case context:
+- target label: ${targetNode.label}
+- target description: ${targetNode.description}
+- central case statement: ${centralStatement}
+
+Question:
+${question}
+
+Answer:
+${answer}`;
+}
+
+function assertRequiredEnv(name) {
+  const value = process.env[name];
+  if (!value) {
+    throw new Error(`${name} is required in .env.local`);
+  }
+  return value;
+}
+
+function validateEnvironment() {
+  const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL");
+  if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
+    throw new Error(`Comparison apparatus refuses localhost fallback. OLLAMA_BASE_URL=${baseUrl}`);
+  }
+  const config = assertConfig();
+  return { baseUrl, modelName: config.OLLAMA_MODEL };
+}
+
+function buildFocusedPlan() {
+  const graph = getGraph();
+  const targetNode = getTargetNode(graph);
+  const prompt = buildFocusedPrompt({
+    targetNode,
+    centralStatement: graph.centralStatement,
+    question: FIXED_QUESTION,
+    answer: FIXED_ANSWER,
+  });
+
+  return {
+    path: "focused",
+    targetNodeId: TARGET_NODE_ID,
+    modelTimingBoundary:
+      "Immediately before provider.generateReconstruction(prompt, modelName) to immediately after it resolves.",
+    inputCharacterCount: prompt.length,
+    prompt,
+    focusedContextSummary: [
+      "target node id and label",
+      "target node description",
+      "central case statement",
+      "fixed question",
+      "fixed answer",
+    ],
+  };
+}
+
+function buildGlobalPlan() {
+  const graph = getGraph();
+  const prompt = buildGraphUpdatePrompt({
+    situationGraph: graph,
+    previousQuestion: FIXED_QUESTION,
+    answer: FIXED_ANSWER,
+  });
+
+  return {
+    path: "global",
+    targetNodeId: TARGET_NODE_ID,
+    modelTimingBoundary:
+      "Immediately before updateCase() calls provider.generateReconstruction(prompt, modelName) to immediately after it resolves inside orchestrator.",
+    endToEndTimingBoundary:
+      "Immediately before updateCase() entry to immediately after full updateCase() result returns.",
+    inputCharacterCount: prompt.length,
+    prompt,
+    graph,
+  };
+}
+
+async function runFocused() {
+  const { modelName } = validateEnvironment();
+  const provider = getProvider();
+  const plan = buildFocusedPlan();
+
+  const startedAt = Date.now();
+  const raw = await provider.generateReconstruction(plan.prompt, modelName);
+  const modelElapsedMs = Date.now() - startedAt;
+
+  const artifact = {
+    path: "focused",
+    targetNodeId: TARGET_NODE_ID,
+    modelName,
+    modelElapsedMs,
+    inputCharacterCount: plan.inputCharacterCount,
+    structuredResult: raw,
+  };
+
+  const artifactPath = path.resolve(
+    "tests/experimental/artifacts/rto-focused-vs-global-focused.json",
+  );
+  await fs.mkdir(path.dirname(artifactPath), { recursive: true });
+  await fs.writeFile(artifactPath, JSON.stringify(artifact, null, 2));
+  console.log(JSON.stringify({ artifactPath, modelElapsedMs, inputCharacterCount: plan.inputCharacterCount }, null, 2));
+}
+
+async function runGlobal() {
+  const { modelName } = validateEnvironment();
+  const plan = buildGlobalPlan();
+
+  const startedAt = Date.now();
+  const result = await runLiveExperiment({
+    graph: plan.graph,
+    previousQuestion: FIXED_QUESTION,
+    answer: FIXED_ANSWER,
+  });
+  const endToEndElapsedMs = Date.now() - startedAt;
+
+  const artifact = {
+    path: "global",
+    targetNodeId: TARGET_NODE_ID,
+    modelName,
+    modelElapsedMs: result.modelElapsedMs,
+    endToEndElapsedMs: result.endToEndElapsedMs ?? endToEndElapsedMs,
+    inputCharacterCount: plan.inputCharacterCount,
+    graphUpdateResultSummary: {
+      userSupportedMeaning: result.userSupportedMeaning,
+      possibleInference: result.possibleInference,
+      proposalValidation: result.proposalValidation,
+      selectedQuestion: result.selectedQuestion,
+    },
+    answerDerivedComparisonExtraction: {
+      observations: result.userSupportedMeaning ? [result.userSupportedMeaning] : "not directly exposed",
+      uncertainties: result.possibleInference ? [result.possibleInference] : "not directly exposed",
+      assumptions: "not directly exposed",
+      relationships: "not directly exposed",
+      newlySurfacedUnknownsOrQuestions: result.selectedQuestion ?? "not directly exposed",
+    },
+  };
+
+  const artifactPath = path.resolve(
+    "tests/experimental/artifacts/rto-focused-vs-global-global.json",
+  );
+  await fs.mkdir(path.dirname(artifactPath), { recursive: true });
+  await fs.writeFile(artifactPath, JSON.stringify(artifact, null, 2));
+  console.log(JSON.stringify({ artifactPath, modelElapsedMs: result.modelElapsedMs, endToEndElapsedMs: result.endToEndElapsedMs ?? endToEndElapsedMs, inputCharacterCount: plan.inputCharacterCount }, null, 2));
+}
+
+function printStaticValidation() {
+  const { modelName } = validateEnvironment();
+  const focused = buildFocusedPlan();
+  const global = buildGlobalPlan();
+
+  const summary = {
+    modelName,
+    fixedTargetNodeId: TARGET_NODE_ID,
+    fixedQuestion: FIXED_QUESTION,
+    fixedAnswer: FIXED_ANSWER,
+    focused: {
+      resolvesTo: "scripts/experimental/rto-focused-answer-deconstruction.mjs-equivalent focused provider path",
+      wholeGraphSupplied: false,
+      inputCharacterCount: focused.inputCharacterCount,
+      modelTimingBoundary: focused.modelTimingBoundary,
+    },
+    global: {
+      resolvesTo: "tests/graph/live-update-experiment-helper.cjs -> updateCase() production path",
+      wholeGraphSupplied: true,
+      inputCharacterCount: global.inputCharacterCount,
+      modelTimingBoundary: global.modelTimingBoundary,
+      endToEndTimingBoundary: global.endToEndTimingBoundary,
+      providerTimingSource: "non-production timed provider wrapper injected through updateCase dependencies.provider",
+    },
+    commands: {
+      focused: "node scripts/experimental/rto-focused-vs-global-comparison.mjs --focused",
+      global: "node scripts/experimental/rto-focused-vs-global-comparison.mjs --global",
+    },
+  };
+
+  console.log(JSON.stringify(summary, null, 2));
+}
+
+const mode = process.argv[2] ?? "--validate";
+
+if (mode === "--focused") {
+  runFocused().catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exit(1);
+  });
+} else if (mode === "--global") {
+  runGlobal().catch((error) => {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exit(1);
+  });
+} else if (mode === "--validate") {
+  printStaticValidation();
+} else {
+  console.error(`Unknown mode: ${mode}`);
+  process.exit(1);
+}