From 952cb442b520e0f73ead6234c4b9d14d5776c26f Mon Sep 17 00:00:00 2001 From: robbond Date: Tue, 18 Aug 2026 18:34:08 +0100 Subject: [PATCH] test(experiment): checkpoint focused vs global comparison apparatus --- .../rto-focused-vs-global-comparison.mjs | 255 ++++++++++++++++++ tests/graph/live-update-experiment-helper.cjs | 23 +- 2 files changed, 277 insertions(+), 1 deletion(-) create mode 100644 scripts/experimental/rto-focused-vs-global-comparison.mjs diff --git a/scripts/experimental/rto-focused-vs-global-comparison.mjs b/scripts/experimental/rto-focused-vs-global-comparison.mjs new file mode 100644 index 0000000..4119a38 --- /dev/null +++ b/scripts/experimental/rto-focused-vs-global-comparison.mjs @@ -0,0 +1,255 @@ +import fs from "fs/promises"; +import path from "path"; +import dotenv from "dotenv"; + +import fixture from "../../tests/fixtures/live-product-launch-update-response.json" with { type: "json" }; +import { buildGraphUpdatePrompt } from "../../lib/graph/prompt-builder.js"; +import { getProvider } from "../../lib/llm/provider.js"; +import { assertConfig } from "../../lib/config.js"; + +import { createRequire } from "module"; +const require = createRequire(import.meta.url); +const { runLiveExperiment } = require("../../tests/graph/live-update-experiment-helper.cjs"); + +dotenv.config({ path: ".env.local" }); + +const TARGET_NODE_ID = "nxmeiab"; +const FIXED_QUESTION = + "What evidence would clarify whether competitors are actively developing similar products and how soon they might release them?"; +const FIXED_ANSWER = + "Two competitors have publicly announced products aimed at the same customer problem. One says it expects a beta within six months, while the other has not announced a release date. We do not yet know how closely either product matches ours."; + +function getGraph() { + return structuredClone(fixture.updatedSituationGraph); +} + +function getTargetNode(graph) { + const targetNode = graph.nodes.find((node) => node.id === TARGET_NODE_ID); + if (!targetNode || targetNode.kind !== "unknown" || targetNode.status === "resolved") { + throw new Error("Fixed target node nxmeiab is not present as unresolved unknown in fixture."); + } + return targetNode; +} + +function buildFocusedPrompt({ targetNode, centralStatement, question, answer }) { + return `You are performing focused answer deconstruction for one explicitly user-chosen investigation. + +Return exactly one JSON object. Return JSON only. + +This is NOT a graph update task. +Do NOT output graph mutations. +Do NOT output selection, ranking, ownership, recommendation, confidence, or next-best-question semantics. +Do NOT include any of these fields: addedNodes, updatedNodes, removedNodes, addedEdges, removedEdges, resolvedNodeIds, activeUnknownNodeId, selectedQuestion. + +Required top-level fields: +- targetNodeId +- observations +- uncertainties +- assumptions +- relationships +- possibleFollowUpQuestions + +Field rules: +- targetNodeId must be exactly "${TARGET_NODE_ID}" +- observations: only statements directly supported by the answer +- uncertainties: only things the answer explicitly leaves unknown or unclear +- assumptions: include only if the answer itself relies on an assumption +- relationships: only direct supported relationships among extracted items, each with { from, to, type, rationale } +- possibleFollowUpQuestions: unresolved questions genuinely exposed by this answer, unranked + +Focused case context: +- target label: ${targetNode.label} +- target description: ${targetNode.description} +- central case statement: ${centralStatement} + +Question: +${question} + +Answer: +${answer}`; +} + +function assertRequiredEnv(name) { + const value = process.env[name]; + if (!value) { + throw new Error(`${name} is required in .env.local`); + } + return value; +} + +function validateEnvironment() { + const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL"); + if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") { + throw new Error(`Comparison apparatus refuses localhost fallback. OLLAMA_BASE_URL=${baseUrl}`); + } + const config = assertConfig(); + return { baseUrl, modelName: config.OLLAMA_MODEL }; +} + +function buildFocusedPlan() { + const graph = getGraph(); + const targetNode = getTargetNode(graph); + const prompt = buildFocusedPrompt({ + targetNode, + centralStatement: graph.centralStatement, + question: FIXED_QUESTION, + answer: FIXED_ANSWER, + }); + + return { + path: "focused", + targetNodeId: TARGET_NODE_ID, + modelTimingBoundary: + "Immediately before provider.generateReconstruction(prompt, modelName) to immediately after it resolves.", + inputCharacterCount: prompt.length, + prompt, + focusedContextSummary: [ + "target node id and label", + "target node description", + "central case statement", + "fixed question", + "fixed answer", + ], + }; +} + +function buildGlobalPlan() { + const graph = getGraph(); + const prompt = buildGraphUpdatePrompt({ + situationGraph: graph, + previousQuestion: FIXED_QUESTION, + answer: FIXED_ANSWER, + }); + + return { + path: "global", + targetNodeId: TARGET_NODE_ID, + modelTimingBoundary: + "Immediately before updateCase() calls provider.generateReconstruction(prompt, modelName) to immediately after it resolves inside orchestrator.", + endToEndTimingBoundary: + "Immediately before updateCase() entry to immediately after full updateCase() result returns.", + inputCharacterCount: prompt.length, + prompt, + graph, + }; +} + +async function runFocused() { + const { modelName } = validateEnvironment(); + const provider = getProvider(); + const plan = buildFocusedPlan(); + + const startedAt = Date.now(); + const raw = await provider.generateReconstruction(plan.prompt, modelName); + const modelElapsedMs = Date.now() - startedAt; + + const artifact = { + path: "focused", + targetNodeId: TARGET_NODE_ID, + modelName, + modelElapsedMs, + inputCharacterCount: plan.inputCharacterCount, + structuredResult: raw, + }; + + const artifactPath = path.resolve( + "tests/experimental/artifacts/rto-focused-vs-global-focused.json", + ); + await fs.mkdir(path.dirname(artifactPath), { recursive: true }); + await fs.writeFile(artifactPath, JSON.stringify(artifact, null, 2)); + console.log(JSON.stringify({ artifactPath, modelElapsedMs, inputCharacterCount: plan.inputCharacterCount }, null, 2)); +} + +async function runGlobal() { + const { modelName } = validateEnvironment(); + const plan = buildGlobalPlan(); + + const startedAt = Date.now(); + const result = await runLiveExperiment({ + graph: plan.graph, + previousQuestion: FIXED_QUESTION, + answer: FIXED_ANSWER, + }); + const endToEndElapsedMs = Date.now() - startedAt; + + const artifact = { + path: "global", + targetNodeId: TARGET_NODE_ID, + modelName, + modelElapsedMs: result.modelElapsedMs, + endToEndElapsedMs: result.endToEndElapsedMs ?? endToEndElapsedMs, + inputCharacterCount: plan.inputCharacterCount, + graphUpdateResultSummary: { + userSupportedMeaning: result.userSupportedMeaning, + possibleInference: result.possibleInference, + proposalValidation: result.proposalValidation, + selectedQuestion: result.selectedQuestion, + }, + answerDerivedComparisonExtraction: { + observations: result.userSupportedMeaning ? [result.userSupportedMeaning] : "not directly exposed", + uncertainties: result.possibleInference ? [result.possibleInference] : "not directly exposed", + assumptions: "not directly exposed", + relationships: "not directly exposed", + newlySurfacedUnknownsOrQuestions: result.selectedQuestion ?? "not directly exposed", + }, + }; + + const artifactPath = path.resolve( + "tests/experimental/artifacts/rto-focused-vs-global-global.json", + ); + await fs.mkdir(path.dirname(artifactPath), { recursive: true }); + await fs.writeFile(artifactPath, JSON.stringify(artifact, null, 2)); + console.log(JSON.stringify({ artifactPath, modelElapsedMs: result.modelElapsedMs, endToEndElapsedMs: result.endToEndElapsedMs ?? endToEndElapsedMs, inputCharacterCount: plan.inputCharacterCount }, null, 2)); +} + +function printStaticValidation() { + const { modelName } = validateEnvironment(); + const focused = buildFocusedPlan(); + const global = buildGlobalPlan(); + + const summary = { + modelName, + fixedTargetNodeId: TARGET_NODE_ID, + fixedQuestion: FIXED_QUESTION, + fixedAnswer: FIXED_ANSWER, + focused: { + resolvesTo: "scripts/experimental/rto-focused-answer-deconstruction.mjs-equivalent focused provider path", + wholeGraphSupplied: false, + inputCharacterCount: focused.inputCharacterCount, + modelTimingBoundary: focused.modelTimingBoundary, + }, + global: { + resolvesTo: "tests/graph/live-update-experiment-helper.cjs -> updateCase() production path", + wholeGraphSupplied: true, + inputCharacterCount: global.inputCharacterCount, + modelTimingBoundary: global.modelTimingBoundary, + endToEndTimingBoundary: global.endToEndTimingBoundary, + providerTimingSource: "non-production timed provider wrapper injected through updateCase dependencies.provider", + }, + commands: { + focused: "node scripts/experimental/rto-focused-vs-global-comparison.mjs --focused", + global: "node scripts/experimental/rto-focused-vs-global-comparison.mjs --global", + }, + }; + + console.log(JSON.stringify(summary, null, 2)); +} + +const mode = process.argv[2] ?? "--validate"; + +if (mode === "--focused") { + runFocused().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }); +} else if (mode === "--global") { + runGlobal().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }); +} else if (mode === "--validate") { + printStaticValidation(); +} else { + console.error(`Unknown mode: ${mode}`); + process.exit(1); +} \ No newline at end of file diff --git a/tests/graph/live-update-experiment-helper.cjs b/tests/graph/live-update-experiment-helper.cjs index 61e4aa5..d52b08b 100644 --- a/tests/graph/live-update-experiment-helper.cjs +++ b/tests/graph/live-update-experiment-helper.cjs @@ -64,7 +64,25 @@ async function runLiveExperiment({ graph, previousQuestion, answer }) { "../../lib/graph/orchestrator.js" ); - const result = await updateCase({ situationGraph: graph, previousQuestion, answer }); + const { getProvider } = await import("../../lib/llm/provider.js"); + const provider = getProvider(); + let modelElapsedMs = null; + + const timedProvider = { + async generateReconstruction(prompt, modelName) { + const startedAt = Date.now(); + const response = await provider.generateReconstruction(prompt, modelName); + modelElapsedMs = Date.now() - startedAt; + return response; + }, + }; + + const endToEndStartedAt = Date.now(); + const result = await updateCase( + { situationGraph: graph, previousQuestion, answer }, + { provider: timedProvider }, + ); + const endToEndElapsedMs = Date.now() - endToEndStartedAt; // Extract the standard checkpoints const answerMeaning = result?.proposal?.answerMeaning || {}; @@ -72,6 +90,9 @@ async function runLiveExperiment({ graph, previousQuestion, answer }) { const supportedCat = _deriveCategory(answerMeaning.userSupportedMeaning || ""); return { + modelElapsedMs, + endToEndElapsedMs, + modelName: model, userSupportedMeaning: answerMeaning.userSupportedMeaning ?? null, possibleInference: answerMeaning.possibleInference ?? null, rawAnswerCategory: rawAnswerCat,