test(experiment): checkpoint focused answer deconstruction

2026-08-18 18:25:35 +01:00
parent 648e1c7a29
commit 2f6c90b027
2 changed files with 237 additions and 0 deletions
@@ -0,0 +1,183 @@
+import fs from "fs/promises";
+import path from "path";
+import dotenv from "dotenv";
+import { z } from "zod";
+
+import fixture from "../../tests/fixtures/live-product-launch-update-response.json" with { type: "json" };
+import { getProvider } from "../../lib/llm/provider.js";
+import { assertConfig } from "../../lib/config.js";
+
+dotenv.config({ path: ".env.local" });
+
+const TARGET_NODE_ID = "nxmeiab";
+const FIXED_QUESTION =
+  "What evidence would clarify whether competitors are actively developing similar products and how soon they might release them?";
+const FIXED_ANSWER =
+  "Two competitors have publicly announced products aimed at the same customer problem. One says it expects a beta within six months, while the other has not announced a release date. We do not yet know how closely either product matches ours.";
+
+const outputSchema = z
+  .object({
+    targetNodeId: z.literal(TARGET_NODE_ID),
+    observations: z.array(z.string()).default([]),
+    uncertainties: z.array(z.string()).default([]),
+    assumptions: z.array(z.string()).default([]),
+    relationships: z
+      .array(
+        z.object({
+          from: z.string().min(1),
+          to: z.string().min(1),
+          type: z.string().min(1),
+          rationale: z.string().min(1),
+        }),
+      )
+      .default([]),
+    possibleFollowUpQuestions: z.array(z.string()).default([]),
+  })
+  .strict()
+  .superRefine((value, ctx) => {
+    const forbiddenFields = [
+      "addedNodes",
+      "updatedNodes",
+      "removedNodes",
+      "addedEdges",
+      "removedEdges",
+      "resolvedNodeIds",
+      "activeUnknownNodeId",
+      "selectedQuestion",
+    ];
+    for (const key of forbiddenFields) {
+      if (key in value) {
+        ctx.addIssue({
+          code: z.ZodIssueCode.custom,
+          message: `Forbidden graph-mutation field present: ${key}`,
+          path: [key],
+        });
+      }
+    }
+  });
+
+function assertRequiredEnv(name) {
+  const value = process.env[name];
+  if (!value) {
+    throw new Error(
+      `Focused apparatus requires ${name}. Set it in .env.local. Found OLLAMA_BASE_URL=${process.env.OLLAMA_BASE_URL ?? "(missing)"}, OLLAMA_MODEL=${process.env.OLLAMA_MODEL ?? "(missing)"}`,
+    );
+  }
+  return value;
+}
+
+function buildFocusedPrompt({ targetNode, centralStatement, question, answer }) {
+  return `You are performing focused answer deconstruction for one explicitly user-chosen investigation.
+
+Return exactly one JSON object. Return JSON only.
+
+This is NOT a graph update task.
+Do NOT output graph mutations.
+Do NOT output selection, ranking, ownership, recommendation, confidence, or next-best-question semantics.
+Do NOT include any of these fields: addedNodes, updatedNodes, removedNodes, addedEdges, removedEdges, resolvedNodeIds, activeUnknownNodeId, selectedQuestion.
+
+Required top-level fields:
+- targetNodeId
+- observations
+- uncertainties
+- assumptions
+- relationships
+- possibleFollowUpQuestions
+
+Field rules:
+- targetNodeId must be exactly "${TARGET_NODE_ID}"
+- observations: only statements directly supported by the answer
+- uncertainties: only things the answer explicitly leaves unknown or unclear
+- assumptions: include only if the answer itself relies on an assumption
+- relationships: only direct supported relationships among extracted items, each with { from, to, type, rationale }
+- possibleFollowUpQuestions: unresolved questions genuinely exposed by this answer, unranked
+
+Focused case context:
+- target label: ${targetNode.label}
+- target description: ${targetNode.description}
+- central case statement: ${centralStatement}
+
+Question:
+${question}
+
+Answer:
+${answer}`;
+}
+
+async function main() {
+  const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL");
+  const config = assertConfig();
+  const modelName = config.OLLAMA_MODEL;
+
+  if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
+    throw new Error(
+      `Focused apparatus refuses localhost fallback. OLLAMA_BASE_URL=${baseUrl}`,
+    );
+  }
+
+  const graph = fixture.updatedSituationGraph;
+  const targetNode = graph.nodes.find((node) => node.id === TARGET_NODE_ID);
+
+  if (!targetNode || targetNode.kind !== "unknown" || targetNode.status === "resolved") {
+    throw new Error("Fixed target node nxmeiab is not present as unresolved unknown in fixture.");
+  }
+
+  const prompt = buildFocusedPrompt({
+    targetNode,
+    centralStatement: graph.centralStatement,
+    question: FIXED_QUESTION,
+    answer: FIXED_ANSWER,
+  });
+
+  const provider = getProvider();
+  const startedAt = Date.now();
+  const raw = await provider.generateReconstruction(prompt, modelName);
+  const elapsedMs = Date.now() - startedAt;
+  const structuredResult = outputSchema.parse(raw);
+
+  const artifact = {
+    targetNodeId: TARGET_NODE_ID,
+    targetLabel: targetNode.label,
+    question: FIXED_QUESTION,
+    answer: FIXED_ANSWER,
+    modelName,
+    elapsedMs,
+    focusedContextSummary: [
+      "target node id and label",
+      "target node description",
+      "central case statement",
+      "fixed question",
+      "fixed answer",
+    ],
+    structuredResult,
+  };
+
+  const artifactPath = path.resolve(
+    "tests/experimental/artifacts/rto-focused-answer-deconstruction-smoke.json",
+  );
+  await fs.mkdir(path.dirname(artifactPath), { recursive: true });
+  await fs.writeFile(artifactPath, JSON.stringify(artifact, null, 2));
+
+  console.log(
+    JSON.stringify(
+      {
+        targetNodeId: artifact.targetNodeId,
+        modelName: artifact.modelName,
+        elapsedMs: artifact.elapsedMs,
+        observations: artifact.structuredResult.observations,
+        uncertainties: artifact.structuredResult.uncertainties,
+        assumptions: artifact.structuredResult.assumptions,
+        relationships: artifact.structuredResult.relationships,
+        possibleFollowUpQuestions:
+          artifact.structuredResult.possibleFollowUpQuestions,
+      },
+      null,
+      2,
+    ),
+  );
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exit(1);
+});
@@ -0,0 +1,54 @@
+{
+  "targetNodeId": "nxmeiab",
+  "targetLabel": "Whether competitors are actively developing similar products and how soon they might release them",
+  "question": "What evidence would clarify whether competitors are actively developing similar products and how soon they might release them?",
+  "answer": "Two competitors have publicly announced products aimed at the same customer problem. One says it expects a beta within six months, while the other has not announced a release date. We do not yet know how closely either product matches ours.",
+  "modelName": "qwen-claude:latest",
+  "elapsedMs": 48805,
+  "focusedContextSummary": [
+    "target node id and label",
+    "target node description",
+    "central case statement",
+    "fixed question",
+    "fixed answer"
+  ],
+  "structuredResult": {
+    "targetNodeId": "nxmeiab",
+    "observations": [
+      "Two competitors have publicly announced products targeting the same customer problem.",
+      "One competitor expects a beta release within six months.",
+      "The second competitor has not announced a release date."
+    ],
+    "uncertainties": [
+      "How closely either competitor's product matches ours.",
+      "When the second competitor will actually release their product."
+    ],
+    "assumptions": [],
+    "relationships": [
+      {
+        "from": "Competitor One",
+        "to": "Beta Release Timeline",
+        "type": "targets",
+        "rationale": "The answer explicitly states one competitor expects a beta within six months."
+      },
+      {
+        "from": "Competitor Two",
+        "to": "Release Schedule",
+        "type": "undisclosed",
+        "rationale": "The answer explicitly states the second competitor has not announced a release date."
+      },
+      {
+        "from": "Both Competitors' Products",
+        "to": "Target Customer Problem",
+        "type": "addresses",
+        "rationale": "The answer directly notes both products are aimed at the same customer problem."
+      }
+    ],
+    "possibleFollowUpQuestions": [
+      "What is the expected timeline for the second competitor's product release?",
+      "How does the feature set and quality of the competitors' current builds compare to ours?",
+      "Will the six-month beta deadline be maintained, or are there signs of development delays?",
+      "Does our pending enterprise customer have any stated interest in evaluating the competing products during their beta phase?"
+    ]
+  }
+}