import fs from "fs/promises"; import path from "path"; import dotenv from "dotenv"; import fixture from "../../tests/fixtures/live-product-launch-update-response.json" with { type: "json" }; import { buildGraphUpdatePrompt } from "../../lib/graph/prompt-builder.js"; import { assertConfig } from "../../lib/config.js"; import { createRequire } from "module"; const require = createRequire(import.meta.url); const { runLiveExperiment } = require("../../tests/graph/live-update-experiment-helper.cjs"); // Non-production diagnostics: an independent request captures raw-response // text and Ollama completion metadata (done, done_reason, prompt_eval_count, // eval_count) before delegating JSON recovery to the inline helper. import { runFocusedDiagnostic } from "./focus-diagnostics-helper.mjs"; dotenv.config({ path: ".env.local" }); const TARGET_NODE_ID = "nxmeiab"; const FIXED_QUESTION = "What evidence would clarify whether competitors are actively developing similar products and how soon they might release them?"; const FIXED_ANSWER = "Two competitors have publicly announced products aimed at the same customer problem. One says it expects a beta within six months, while the other has not announced a release date. We do not yet know how closely either product matches ours."; function getGraph() { return structuredClone(fixture.updatedSituationGraph); } function getTargetNode(graph) { const targetNode = graph.nodes.find((node) => node.id === TARGET_NODE_ID); if (!targetNode || targetNode.kind !== "unknown" || targetNode.status === "resolved") { throw new Error("Fixed target node nxmeiab is not present as unresolved unknown in fixture."); } return targetNode; } function buildFocusedPrompt({ targetNode, centralStatement, question, answer }) { return `You are performing focused answer deconstruction for one explicitly user-chosen investigation. Return exactly one JSON object. Return JSON only. This is NOT a graph update task. Do NOT output graph mutations. Do NOT output selection, ranking, ownership, recommendation, confidence, or next-best-question semantics. Do NOT include any of these fields: addedNodes, updatedNodes, removedNodes, addedEdges, removedEdges, resolvedNodeIds, activeUnknownNodeId, selectedQuestion. Required top-level fields: - targetNodeId - observations - uncertainties - assumptions - relationships - possibleFollowUpQuestions Field rules: - targetNodeId must be exactly "${TARGET_NODE_ID}" - observations: only statements directly supported by the answer - uncertainties: only things the answer explicitly leaves unknown or unclear - assumptions: include only if the answer itself relies on an assumption - relationships: only direct supported relationships among extracted items, each with { from, to, type, rationale } - possibleFollowUpQuestions: unresolved questions genuinely exposed by this answer, unranked Focused case context: - target label: ${targetNode.label} - target description: ${targetNode.description} - central case statement: ${centralStatement} Question: ${question} Answer: ${answer}`; } function assertRequiredEnv(name) { const value = process.env[name]; if (!value) { throw new Error(`${name} is required in .env.local`); } return value; } function validateEnvironment() { const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL"); if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") { throw new Error(`Comparison apparatus refuses localhost fallback. OLLAMA_BASE_URL=${baseUrl}`); } const config = assertConfig(); return { baseUrl, modelName: config.OLLAMA_MODEL }; } function buildFocusedPlan() { const graph = getGraph(); const targetNode = getTargetNode(graph); const prompt = buildFocusedPrompt({ targetNode, centralStatement: graph.centralStatement, question: FIXED_QUESTION, answer: FIXED_ANSWER, }); return { path: "focused", targetNodeId: TARGET_NODE_ID, modelTimingBoundary: "Immediately before provider.generateReconstruction(prompt, modelName) to immediately after it resolves.", inputCharacterCount: prompt.length, prompt, focusedContextSummary: [ "target node id and label", "target node description", "central case statement", "fixed question", "fixed answer", ], }; } function buildGlobalPlan() { const graph = getGraph(); const prompt = buildGraphUpdatePrompt({ situationGraph: graph, previousQuestion: FIXED_QUESTION, answer: FIXED_ANSWER, }); return { path: "global", targetNodeId: TARGET_NODE_ID, modelTimingBoundary: "Immediately before updateCase() calls provider.generateReconstruction(prompt, modelName) to immediately after it resolves inside orchestrator.", endToEndTimingBoundary: "Immediately before updateCase() entry to immediately after full updateCase() result returns.", inputCharacterCount: prompt.length, prompt, graph, }; } async function runFocused() { const { baseUrl, modelName } = validateEnvironment(); const plan = buildFocusedPlan(); // ---- non-production diagnostics wrapper ---- // Independent request captures raw response + completion metadata. // The diagnostics helper mirrors the production JSON-recovery logic inline // (since recoverJson is not exported) and delegates parse back to that // implementation. The comparison runner only observes result/error objects. const diagResult = await runFocusedDiagnostic({ baseUrl, modelName, prompt: plan.prompt }); try { if (diagResult.error) { throw diagResult.error; } // Successful parse via production helper — proceed to standard artifact const raw = diagResult.parsedResult; const artifact = { path: "focused", targetNodeId: TARGET_NODE_ID, modelName, modelElapsedMs: diagResult.diagnostics.modelElapsedMs, inputCharacterCount: plan.inputCharacterCount, structuredResult: raw, }; const artifactPath = path.resolve( "tests/experimental/artifacts/rto-focused-vs-global-focused.json", ); await fs.mkdir(path.dirname(artifactPath), { recursive: true }); await fs.writeFile(artifactPath, JSON.stringify(artifact, null, 2)); console.log(JSON.stringify({ artifactPath, modelElapsedMs: diagResult.diagnostics.modelElapsedMs, inputCharacterCount: plan.inputCharacterCount }, null, 2)); return; } catch (err) { const focusedDiagnostics = err.focusedDiagnostics ?? null; if (!focusedDiagnostics) { // Should not happen — every DiagnosticError carries diagnostics. console.error("Unexpected: no diagnostics captured before parse failure."); process.exit(1); } const errorArtifact = { path: "focused", modelName: focusedDiagnostics.apiUsed === "/api/chat" ? modelName : modelName, inputCharacterCount: plan.inputCharacterCount, rawResponseText: focusedDiagnostics.rawResponseText?.slice(0, 4096), rawResponseCharacterCount: focusedDiagnostics.rawResponseCharacterCount, done: focusedDiagnostics.done, doneReason: focusedDiagnostics.doneReason, promptEvalCount: focusedDiagnostics.promptEvalCount, evalCount: focusedDiagnostics.evalCount, errorType: err.name, errorMessage: err.message.slice(0, 4096), }; const artifactPath = path.resolve( "tests/experimental/artifacts/rto-focused-vs-global-focused-failure.json", ); await fs.mkdir(path.dirname(artifactPath), { recursive: true }); await fs.writeFile(artifactPath, JSON.stringify(errorArtifact, null, 2)); console.log(JSON.stringify({ artifactPath, errorType: err.name, rawResponseCharacterCount: focusedDiagnostics.rawResponseCharacterCount, done: focusedDiagnostics.done, doneReason: focusedDiagnostics.doneReason, }, null, 2)); } } async function runGlobal() { const { modelName } = validateEnvironment(); const plan = buildGlobalPlan(); const startedAt = Date.now(); const result = await runLiveExperiment({ graph: plan.graph, previousQuestion: FIXED_QUESTION, answer: FIXED_ANSWER, }); const endToEndElapsedMs = Date.now() - startedAt; const artifact = { path: "global", targetNodeId: TARGET_NODE_ID, modelName, modelElapsedMs: result.modelElapsedMs, providerGenerateCalls: result.providerGenerateCalls ?? 0, providerTimingCaptured: typeof result.modelElapsedMs === "number", endToEndElapsedMs: result.endToEndElapsedMs ?? endToEndElapsedMs, inputCharacterCount: plan.inputCharacterCount, graphUpdateResultSummary: { userSupportedMeaning: result.userSupportedMeaning, possibleInference: result.possibleInference, proposalValidation: result.proposalValidation, selectedQuestion: result.selectedQuestion, }, answerDerivedComparisonExtraction: { observations: result.userSupportedMeaning ? [result.userSupportedMeaning] : "not directly exposed", uncertainties: result.possibleInference ? [result.possibleInference] : "not directly exposed", assumptions: "not directly exposed", relationships: "not directly exposed", newlySurfacedUnknownsOrQuestions: result.selectedQuestion ?? "not directly exposed", }, }; const artifactPath = path.resolve( "tests/experimental/artifacts/rto-focused-vs-global-global.json", ); await fs.mkdir(path.dirname(artifactPath), { recursive: true }); await fs.writeFile(artifactPath, JSON.stringify(artifact, null, 2)); console.log(JSON.stringify({ artifactPath, modelElapsedMs: result.modelElapsedMs, endToEndElapsedMs: result.endToEndElapsedMs ?? endToEndElapsedMs, inputCharacterCount: plan.inputCharacterCount }, null, 2)); } function printStaticValidation() { const { modelName } = validateEnvironment(); const focused = buildFocusedPlan(); const global = buildGlobalPlan(); const summary = { modelName, fixedTargetNodeId: TARGET_NODE_ID, fixedQuestion: FIXED_QUESTION, fixedAnswer: FIXED_ANSWER, focused: { resolvesTo: "scripts/experimental/rto-focused-vs-global-comparison-equivalent focused provider path", wholeGraphSupplied: false, inputCharacterCount: focused.inputCharacterCount, modelTimingBoundary: focused.modelTimingBoundary, }, global: { resolvesTo: "tests/graph/live-update-experiment-helper.cjs -> updateCase() production path", wholeGraphSupplied: true, inputCharacterCount: global.inputCharacterCount, modelTimingBoundary: global.modelTimingBoundary, endToEndTimingBoundary: global.endToEndTimingBoundary, providerTimingSource: "non-production timed provider wrapper injected through updateCase dependencies.provider", }, commands: { focused: "node scripts/experimental/rto-focused-vs-global-comparison.mjs --focused", global: "node scripts/experimental/rto-focused-vs-global-comparison.mjs --global", }, }; console.log(JSON.stringify(summary, null, 2)); } const mode = process.argv[2] ?? "--validate"; if (mode === "--focused") { runFocused().catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exit(1); }); } else if (mode === "--global") { runGlobal().catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exit(1); }); } else if (mode === "--validate") { printStaticValidation(); } else { console.error(`Unknown mode: ${mode}`); process.exit(1); }