From 2f6c90b0274bc3cc806711e9b04bff16dc83b076 Mon Sep 17 00:00:00 2001 From: robbond Date: Tue, 18 Aug 2026 18:25:35 +0100 Subject: [PATCH] test(experiment): checkpoint focused answer deconstruction --- .../rto-focused-answer-deconstruction.mjs | 183 ++++++++++++++++++ ...o-focused-answer-deconstruction-smoke.json | 54 ++++++ 2 files changed, 237 insertions(+) create mode 100644 scripts/experimental/rto-focused-answer-deconstruction.mjs create mode 100644 tests/experimental/artifacts/rto-focused-answer-deconstruction-smoke.json diff --git a/scripts/experimental/rto-focused-answer-deconstruction.mjs b/scripts/experimental/rto-focused-answer-deconstruction.mjs new file mode 100644 index 0000000..5d606ba --- /dev/null +++ b/scripts/experimental/rto-focused-answer-deconstruction.mjs @@ -0,0 +1,183 @@ +import fs from "fs/promises"; +import path from "path"; +import dotenv from "dotenv"; +import { z } from "zod"; + +import fixture from "../../tests/fixtures/live-product-launch-update-response.json" with { type: "json" }; +import { getProvider } from "../../lib/llm/provider.js"; +import { assertConfig } from "../../lib/config.js"; + +dotenv.config({ path: ".env.local" }); + +const TARGET_NODE_ID = "nxmeiab"; +const FIXED_QUESTION = + "What evidence would clarify whether competitors are actively developing similar products and how soon they might release them?"; +const FIXED_ANSWER = + "Two competitors have publicly announced products aimed at the same customer problem. One says it expects a beta within six months, while the other has not announced a release date. We do not yet know how closely either product matches ours."; + +const outputSchema = z + .object({ + targetNodeId: z.literal(TARGET_NODE_ID), + observations: z.array(z.string()).default([]), + uncertainties: z.array(z.string()).default([]), + assumptions: z.array(z.string()).default([]), + relationships: z + .array( + z.object({ + from: z.string().min(1), + to: z.string().min(1), + type: z.string().min(1), + rationale: z.string().min(1), + }), + ) + .default([]), + possibleFollowUpQuestions: z.array(z.string()).default([]), + }) + .strict() + .superRefine((value, ctx) => { + const forbiddenFields = [ + "addedNodes", + "updatedNodes", + "removedNodes", + "addedEdges", + "removedEdges", + "resolvedNodeIds", + "activeUnknownNodeId", + "selectedQuestion", + ]; + for (const key of forbiddenFields) { + if (key in value) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Forbidden graph-mutation field present: ${key}`, + path: [key], + }); + } + } + }); + +function assertRequiredEnv(name) { + const value = process.env[name]; + if (!value) { + throw new Error( + `Focused apparatus requires ${name}. Set it in .env.local. Found OLLAMA_BASE_URL=${process.env.OLLAMA_BASE_URL ?? "(missing)"}, OLLAMA_MODEL=${process.env.OLLAMA_MODEL ?? "(missing)"}`, + ); + } + return value; +} + +function buildFocusedPrompt({ targetNode, centralStatement, question, answer }) { + return `You are performing focused answer deconstruction for one explicitly user-chosen investigation. + +Return exactly one JSON object. Return JSON only. + +This is NOT a graph update task. +Do NOT output graph mutations. +Do NOT output selection, ranking, ownership, recommendation, confidence, or next-best-question semantics. +Do NOT include any of these fields: addedNodes, updatedNodes, removedNodes, addedEdges, removedEdges, resolvedNodeIds, activeUnknownNodeId, selectedQuestion. + +Required top-level fields: +- targetNodeId +- observations +- uncertainties +- assumptions +- relationships +- possibleFollowUpQuestions + +Field rules: +- targetNodeId must be exactly "${TARGET_NODE_ID}" +- observations: only statements directly supported by the answer +- uncertainties: only things the answer explicitly leaves unknown or unclear +- assumptions: include only if the answer itself relies on an assumption +- relationships: only direct supported relationships among extracted items, each with { from, to, type, rationale } +- possibleFollowUpQuestions: unresolved questions genuinely exposed by this answer, unranked + +Focused case context: +- target label: ${targetNode.label} +- target description: ${targetNode.description} +- central case statement: ${centralStatement} + +Question: +${question} + +Answer: +${answer}`; +} + +async function main() { + const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL"); + const config = assertConfig(); + const modelName = config.OLLAMA_MODEL; + + if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") { + throw new Error( + `Focused apparatus refuses localhost fallback. OLLAMA_BASE_URL=${baseUrl}`, + ); + } + + const graph = fixture.updatedSituationGraph; + const targetNode = graph.nodes.find((node) => node.id === TARGET_NODE_ID); + + if (!targetNode || targetNode.kind !== "unknown" || targetNode.status === "resolved") { + throw new Error("Fixed target node nxmeiab is not present as unresolved unknown in fixture."); + } + + const prompt = buildFocusedPrompt({ + targetNode, + centralStatement: graph.centralStatement, + question: FIXED_QUESTION, + answer: FIXED_ANSWER, + }); + + const provider = getProvider(); + const startedAt = Date.now(); + const raw = await provider.generateReconstruction(prompt, modelName); + const elapsedMs = Date.now() - startedAt; + const structuredResult = outputSchema.parse(raw); + + const artifact = { + targetNodeId: TARGET_NODE_ID, + targetLabel: targetNode.label, + question: FIXED_QUESTION, + answer: FIXED_ANSWER, + modelName, + elapsedMs, + focusedContextSummary: [ + "target node id and label", + "target node description", + "central case statement", + "fixed question", + "fixed answer", + ], + structuredResult, + }; + + const artifactPath = path.resolve( + "tests/experimental/artifacts/rto-focused-answer-deconstruction-smoke.json", + ); + await fs.mkdir(path.dirname(artifactPath), { recursive: true }); + await fs.writeFile(artifactPath, JSON.stringify(artifact, null, 2)); + + console.log( + JSON.stringify( + { + targetNodeId: artifact.targetNodeId, + modelName: artifact.modelName, + elapsedMs: artifact.elapsedMs, + observations: artifact.structuredResult.observations, + uncertainties: artifact.structuredResult.uncertainties, + assumptions: artifact.structuredResult.assumptions, + relationships: artifact.structuredResult.relationships, + possibleFollowUpQuestions: + artifact.structuredResult.possibleFollowUpQuestions, + }, + null, + 2, + ), + ); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); \ No newline at end of file diff --git a/tests/experimental/artifacts/rto-focused-answer-deconstruction-smoke.json b/tests/experimental/artifacts/rto-focused-answer-deconstruction-smoke.json new file mode 100644 index 0000000..2ff9f84 --- /dev/null +++ b/tests/experimental/artifacts/rto-focused-answer-deconstruction-smoke.json @@ -0,0 +1,54 @@ +{ + "targetNodeId": "nxmeiab", + "targetLabel": "Whether competitors are actively developing similar products and how soon they might release them", + "question": "What evidence would clarify whether competitors are actively developing similar products and how soon they might release them?", + "answer": "Two competitors have publicly announced products aimed at the same customer problem. One says it expects a beta within six months, while the other has not announced a release date. We do not yet know how closely either product matches ours.", + "modelName": "qwen-claude:latest", + "elapsedMs": 48805, + "focusedContextSummary": [ + "target node id and label", + "target node description", + "central case statement", + "fixed question", + "fixed answer" + ], + "structuredResult": { + "targetNodeId": "nxmeiab", + "observations": [ + "Two competitors have publicly announced products targeting the same customer problem.", + "One competitor expects a beta release within six months.", + "The second competitor has not announced a release date." + ], + "uncertainties": [ + "How closely either competitor's product matches ours.", + "When the second competitor will actually release their product." + ], + "assumptions": [], + "relationships": [ + { + "from": "Competitor One", + "to": "Beta Release Timeline", + "type": "targets", + "rationale": "The answer explicitly states one competitor expects a beta within six months." + }, + { + "from": "Competitor Two", + "to": "Release Schedule", + "type": "undisclosed", + "rationale": "The answer explicitly states the second competitor has not announced a release date." + }, + { + "from": "Both Competitors' Products", + "to": "Target Customer Problem", + "type": "addresses", + "rationale": "The answer directly notes both products are aimed at the same customer problem." + } + ], + "possibleFollowUpQuestions": [ + "What is the expected timeline for the second competitor's product release?", + "How does the feature set and quality of the competitors' current builds compare to ours?", + "Will the six-month beta deadline be maintained, or are there signs of development delays?", + "Does our pending enterprise customer have any stated interest in evaluating the competing products during their beta phase?" + ] + } +} \ No newline at end of file