From a93b6798cc47d82a0026258ba8f8bea6706e0d98 Mon Sep 17 00:00:00 2001 From: robbond Date: Sun, 6 Sep 2026 16:21:26 +0100 Subject: [PATCH] fix(confidence-engine): supply focused deconstruction schema --- .../deconstruct/route.js | 12 +++- docs/current-handoff.md | 7 +++ lib/graph/focused-investigation.js | 14 +++++ lib/llm/provider.js | 8 +-- tests/focused-deconstruct-boundary.test.js | 55 +++++++++++++++++-- tests/llm/provider.test.js | 31 +++++++++-- 6 files changed, 110 insertions(+), 17 deletions(-) diff --git a/app/api/focused-investigation/deconstruct/route.js b/app/api/focused-investigation/deconstruct/route.js index 528c96f..008fb2a 100644 --- a/app/api/focused-investigation/deconstruct/route.js +++ b/app/api/focused-investigation/deconstruct/route.js @@ -1,5 +1,9 @@ import { getProvider } from "@/lib/llm/provider"; -import { buildFocusedDeconstructPrompt, validateFocusedDeconstructSchema } from "@/lib/graph/focused-investigation"; +import { + buildFocusedDeconstructPrompt, + focusedDeconstructJsonSchema, + validateFocusedDeconstructSchema, +} from "@/lib/graph/focused-investigation"; export async function POST(request) { try { @@ -52,7 +56,11 @@ export async function POST(request) { const provider = getProvider(); const startedAt = Date.now(); - const wrapper = await provider.generateReconstruction(prompt, process.env.OLLAMA_MODEL); + const wrapper = await provider.generateReconstruction( + prompt, + process.env.OLLAMA_MODEL, + focusedDeconstructJsonSchema, + ); const elapsedMs = Date.now() - startedAt; // Unwrap the semantic deconstruction from the provider envelope. diff --git a/docs/current-handoff.md b/docs/current-handoff.md index 88246e7..a670d47 100644 --- a/docs/current-handoff.md +++ b/docs/current-handoff.md @@ -156,6 +156,13 @@ - The first reconstruction-only Terra request reached inference successfully, but native-fetch parsing relied on SDK-only `output_text` and could not extract the raw response. The provider now reads documented `output[].content[].output_text` parts in response order while retaining the convenience-property path. - No tool-call assumption was introduced; canonical schema, prompt, transport compatibility, and reasoning remain unchanged. Zero live calls occurred during this correction; next boundary is exactly one live Terra reconstruction with no retry and no Ollama call. +## Focused-deconstruction structured-output transport + +- Confirmed mismatch: the focused route requested and validated its six-field deconstruction contract while Ollama `/api/chat` was constrained to the initial reconstruction schema. +- The focused route now supplies `focusedDeconstructJsonSchema`; `generateReconstruction()` accepts it as an optional chat-format argument, while initial reconstruction callers retain the default `reconstructionJsonSchema`. +- Provider tests prove default and alternate schema transport. Focused route tests model the real provider wrapper, preserve `wrapper.response` unwrapping, and verify the schema argument without module/mock-state contamination. +- Previous failed live focused-deconstruction observations remain invalid semantic evidence. Zero live calls occurred during implementation and apparatus correction. Next boundary: exactly one substantive compound-answer live observation through the corrected production route. + ## Current product architecture Three distinct routes, not a single page: diff --git a/lib/graph/focused-investigation.js b/lib/graph/focused-investigation.js index 7207ce2..a7aa827 100644 --- a/lib/graph/focused-investigation.js +++ b/lib/graph/focused-investigation.js @@ -9,6 +9,20 @@ const FOCUSED_ANSWER_SCHEMA_FIELDS = [ "possibleFollowUpQuestions", ]; +export const focusedDeconstructJsonSchema = { + type: "object", + properties: { + targetNodeId: { type: "string" }, + observations: { type: "array", items: { type: "string" } }, + uncertainties: { type: "array", items: { type: "string" } }, + assumptions: { type: "array", items: { type: "string" } }, + relationships: { type: "array", items: { type: "object" } }, + possibleFollowUpQuestions: { type: "array", items: { type: "string" } }, + }, + required: FOCUSED_ANSWER_SCHEMA_FIELDS, + additionalProperties: false, +}; + const FORBIDDEN_GRAPH_MUTATION_FIELDS = [ "addedNodes", "updatedNodes", diff --git a/lib/llm/provider.js b/lib/llm/provider.js index fd8db45..50e8b9b 100644 --- a/lib/llm/provider.js +++ b/lib/llm/provider.js @@ -96,7 +96,7 @@ function extractOpenAIResponseText(data) { } let _chatSupported = null; -const reconstructionJsonSchema = z.toJSONSchema(reconstructionV2Schema); +export const reconstructionJsonSchema = z.toJSONSchema(reconstructionV2Schema); const openAIReconstructionJsonSchema = createOpenAIStrictSchema( reconstructionJsonSchema, reconstructionV2Schema, @@ -251,7 +251,7 @@ async function detectChatSupport(baseUrl, modelName) { } class OllamaLlmProvider { - async generateReconstruction(scenario, modelName) { + async generateReconstruction(scenario, modelName, outputSchema) { // scenario is ALREADY a fully-built prompt text (built by analyseScenario). // Do NOT call buildPrompt() again — that would double-wrap the prompt. const prompt = scenario; @@ -279,7 +279,7 @@ class OllamaLlmProvider { providerExecution.chatCapabilityDetected = chatSupported; // ================================================================ - // Step 2: Try /api/chat if supported with the reconstruction schema + // Step 2: Try /api/chat if supported with the supplied or reconstruction schema // ================================================================ if (chatSupported) { try { @@ -295,7 +295,7 @@ class OllamaLlmProvider { model: modelName, messages: [{ role: "user", content: prompt }], stream: false, - format: reconstructionJsonSchema, + format: outputSchema ?? reconstructionJsonSchema, }), signal: controller.signal, }); diff --git a/tests/focused-deconstruct-boundary.test.js b/tests/focused-deconstruct-boundary.test.js index 6e028ea..66db5ee 100644 --- a/tests/focused-deconstruct-boundary.test.js +++ b/tests/focused-deconstruct-boundary.test.js @@ -6,7 +6,8 @@ * API response targetNodeId regardless of what the model returns. */ -import { describe, it, expect, vi } from "vitest"; +import { afterEach, beforeEach, describe, it, expect, vi } from "vitest"; +import { focusedDeconstructJsonSchema } from "@/lib/graph/focused-investigation"; // ── helpers ────────────────────────────────────────────────────────────── @@ -36,6 +37,14 @@ function makeMockProvider(inventedTargetNodeId) { // ── Boundary test ──────────────────────────────────────────────────────── describe("focused-deconstruct targetNodeId identity boundary", () => { + beforeEach(() => { + vi.resetModules(); + }); + + afterEach(() => { + vi.doUnmock("@/lib/llm/provider"); + }); + it("request targetNodeId overrides model-invented targetNodeId", async () => { const requestTargetNodeId = "nk04xvk"; // original graph node ID const inventedModelId = "invented-model-id"; @@ -73,6 +82,45 @@ describe("focused-deconstruct targetNodeId identity boundary", () => { expect(json.targetNodeId).not.toBe(inventedModelId); }); + it("supplies the focused-deconstruction schema through the provider seam", async () => { + const generateReconstruction = vi.fn().mockResolvedValue({ + response: { + targetNodeId: "model-id", + observations: [], + uncertainties: [], + assumptions: [], + relationships: [], + possibleFollowUpQuestions: [], + }, + providerApiPath: "/api/chat", + providerExecution: { chatRequestAttempted: true }, + }); + vi.doMock("@/lib/llm/provider", () => ({ + getProvider: () => ({ generateReconstruction }), + })); + const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js"); + + const response = await POST(new Request("http://localhost/api/focused-investigation/deconstruct", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + targetNodeId: "node-id", + targetLabel: "label", + targetDescription: "description", + centralStatement: "central statement", + question: "question?", + answer: "answer.", + }), + })); + + expect(response.status).toBe(200); + expect(generateReconstruction).toHaveBeenCalledWith( + expect.any(String), + process.env.OLLAMA_MODEL, + focusedDeconstructJsonSchema, + ); + }); + it("semantic fields pass through unchanged from model", async () => { const mockObs = ["doc is minimal", "processes in founder's head"]; const mockUnc = ["whether formal docs can capture tacit knowledge"]; @@ -157,9 +205,6 @@ describe("focused-deconstruct targetNodeId identity boundary", () => { }); it("full identity path: request → response → contribution", async () => { - // Reset modules to avoid mock leakage from earlier tests - vi.resetModules(); - const originalNodeId = "nk04xvk"; const modelInventedId = "investigation_node_responsibility_distribution_autonomy"; @@ -231,8 +276,6 @@ describe("focused-deconstruct targetNodeId identity boundary", () => { }); it("provider envelope fields do not leak into API response", async () => { - vi.resetModules(); - vi.doMock("@/lib/llm/provider", () => ({ getProvider: () => ({ generateReconstruction: vi.fn().mockResolvedValue({ diff --git a/tests/llm/provider.test.js b/tests/llm/provider.test.js index a861686..1a1c2b3 100644 --- a/tests/llm/provider.test.js +++ b/tests/llm/provider.test.js @@ -5,6 +5,7 @@ import { createOpenAIReconstructionProvider, getProvider, normaliseOpenAITransportResponse, + reconstructionJsonSchema, } from "@/lib/llm/provider.js"; import { reconstructionV2Schema } from "@/lib/reconstruction/schema.js"; import { z } from "zod"; @@ -40,11 +41,7 @@ describe("OllamaLlmProvider chat capability detection", () => { messages: [{ role: "user", content: "prompt" }], stream: false, }); - expect(chatRequest.format).toBeTypeOf("object"); - expect(chatRequest.format).not.toBe("json"); - const formatText = JSON.stringify(chatRequest.format); - expect(formatText).toContain("relationship"); - expect(formatText).toContain("evidenceType"); + expect(chatRequest.format).toEqual(reconstructionJsonSchema); expect(result).toMatchObject({ response: {}, providerApiPath: "/api/chat", @@ -63,6 +60,30 @@ describe("OllamaLlmProvider chat capability detection", () => { } }); + it("uses a supplied structured-output schema for the chat request", async () => { + const originalBaseUrl = process.env.OLLAMA_BASE_URL; + const outputSchema = { + type: "object", + properties: { focused: { type: "string" } }, + required: ["focused"], + }; + const fetchSpy = vi.fn() + .mockResolvedValueOnce({ ok: true, text: async () => "" }) + .mockResolvedValueOnce({ ok: true, json: async () => ({ message: { content: "{}" } }) }); + vi.stubGlobal("fetch", fetchSpy); + process.env.OLLAMA_BASE_URL = "http://ollama.test"; + + try { + __resetChatSupportForTests(); + await getProvider().generateReconstruction("prompt", "configured-model", outputSchema); + expect(JSON.parse(fetchSpy.mock.calls[1][1].body).format).toEqual(outputSchema); + } finally { + vi.unstubAllGlobals(); + if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL; + else process.env.OLLAMA_BASE_URL = originalBaseUrl; + } + }); + it("reports chat-skipped generate fallback execution", async () => { const originalBaseUrl = process.env.OLLAMA_BASE_URL; const fetchSpy = vi.fn()