fix(confidence-engine): supply focused deconstruction schema

This commit is contained in:
2026-09-06 16:21:26 +01:00
parent 188dd04ab9
commit a93b6798cc
6 changed files with 110 additions and 17 deletions
@@ -1,5 +1,9 @@
import { getProvider } from "@/lib/llm/provider"; import { getProvider } from "@/lib/llm/provider";
import { buildFocusedDeconstructPrompt, validateFocusedDeconstructSchema } from "@/lib/graph/focused-investigation"; import {
buildFocusedDeconstructPrompt,
focusedDeconstructJsonSchema,
validateFocusedDeconstructSchema,
} from "@/lib/graph/focused-investigation";
export async function POST(request) { export async function POST(request) {
try { try {
@@ -52,7 +56,11 @@ export async function POST(request) {
const provider = getProvider(); const provider = getProvider();
const startedAt = Date.now(); const startedAt = Date.now();
const wrapper = await provider.generateReconstruction(prompt, process.env.OLLAMA_MODEL); const wrapper = await provider.generateReconstruction(
prompt,
process.env.OLLAMA_MODEL,
focusedDeconstructJsonSchema,
);
const elapsedMs = Date.now() - startedAt; const elapsedMs = Date.now() - startedAt;
// Unwrap the semantic deconstruction from the provider envelope. // Unwrap the semantic deconstruction from the provider envelope.
+7
View File
@@ -156,6 +156,13 @@
- The first reconstruction-only Terra request reached inference successfully, but native-fetch parsing relied on SDK-only `output_text` and could not extract the raw response. The provider now reads documented `output[].content[].output_text` parts in response order while retaining the convenience-property path. - The first reconstruction-only Terra request reached inference successfully, but native-fetch parsing relied on SDK-only `output_text` and could not extract the raw response. The provider now reads documented `output[].content[].output_text` parts in response order while retaining the convenience-property path.
- No tool-call assumption was introduced; canonical schema, prompt, transport compatibility, and reasoning remain unchanged. Zero live calls occurred during this correction; next boundary is exactly one live Terra reconstruction with no retry and no Ollama call. - No tool-call assumption was introduced; canonical schema, prompt, transport compatibility, and reasoning remain unchanged. Zero live calls occurred during this correction; next boundary is exactly one live Terra reconstruction with no retry and no Ollama call.
## Focused-deconstruction structured-output transport
- Confirmed mismatch: the focused route requested and validated its six-field deconstruction contract while Ollama `/api/chat` was constrained to the initial reconstruction schema.
- The focused route now supplies `focusedDeconstructJsonSchema`; `generateReconstruction()` accepts it as an optional chat-format argument, while initial reconstruction callers retain the default `reconstructionJsonSchema`.
- Provider tests prove default and alternate schema transport. Focused route tests model the real provider wrapper, preserve `wrapper.response` unwrapping, and verify the schema argument without module/mock-state contamination.
- Previous failed live focused-deconstruction observations remain invalid semantic evidence. Zero live calls occurred during implementation and apparatus correction. Next boundary: exactly one substantive compound-answer live observation through the corrected production route.
## Current product architecture ## Current product architecture
Three distinct routes, not a single page: Three distinct routes, not a single page:
+14
View File
@@ -9,6 +9,20 @@ const FOCUSED_ANSWER_SCHEMA_FIELDS = [
"possibleFollowUpQuestions", "possibleFollowUpQuestions",
]; ];
export const focusedDeconstructJsonSchema = {
type: "object",
properties: {
targetNodeId: { type: "string" },
observations: { type: "array", items: { type: "string" } },
uncertainties: { type: "array", items: { type: "string" } },
assumptions: { type: "array", items: { type: "string" } },
relationships: { type: "array", items: { type: "object" } },
possibleFollowUpQuestions: { type: "array", items: { type: "string" } },
},
required: FOCUSED_ANSWER_SCHEMA_FIELDS,
additionalProperties: false,
};
const FORBIDDEN_GRAPH_MUTATION_FIELDS = [ const FORBIDDEN_GRAPH_MUTATION_FIELDS = [
"addedNodes", "addedNodes",
"updatedNodes", "updatedNodes",
+4 -4
View File
@@ -96,7 +96,7 @@ function extractOpenAIResponseText(data) {
} }
let _chatSupported = null; let _chatSupported = null;
const reconstructionJsonSchema = z.toJSONSchema(reconstructionV2Schema); export const reconstructionJsonSchema = z.toJSONSchema(reconstructionV2Schema);
const openAIReconstructionJsonSchema = createOpenAIStrictSchema( const openAIReconstructionJsonSchema = createOpenAIStrictSchema(
reconstructionJsonSchema, reconstructionJsonSchema,
reconstructionV2Schema, reconstructionV2Schema,
@@ -251,7 +251,7 @@ async function detectChatSupport(baseUrl, modelName) {
} }
class OllamaLlmProvider { class OllamaLlmProvider {
async generateReconstruction(scenario, modelName) { async generateReconstruction(scenario, modelName, outputSchema) {
// scenario is ALREADY a fully-built prompt text (built by analyseScenario). // scenario is ALREADY a fully-built prompt text (built by analyseScenario).
// Do NOT call buildPrompt() again — that would double-wrap the prompt. // Do NOT call buildPrompt() again — that would double-wrap the prompt.
const prompt = scenario; const prompt = scenario;
@@ -279,7 +279,7 @@ class OllamaLlmProvider {
providerExecution.chatCapabilityDetected = chatSupported; providerExecution.chatCapabilityDetected = chatSupported;
// ================================================================ // ================================================================
// Step 2: Try /api/chat if supported with the reconstruction schema // Step 2: Try /api/chat if supported with the supplied or reconstruction schema
// ================================================================ // ================================================================
if (chatSupported) { if (chatSupported) {
try { try {
@@ -295,7 +295,7 @@ class OllamaLlmProvider {
model: modelName, model: modelName,
messages: [{ role: "user", content: prompt }], messages: [{ role: "user", content: prompt }],
stream: false, stream: false,
format: reconstructionJsonSchema, format: outputSchema ?? reconstructionJsonSchema,
}), }),
signal: controller.signal, signal: controller.signal,
}); });
+49 -6
View File
@@ -6,7 +6,8 @@
* API response targetNodeId regardless of what the model returns. * API response targetNodeId regardless of what the model returns.
*/ */
import { describe, it, expect, vi } from "vitest"; import { afterEach, beforeEach, describe, it, expect, vi } from "vitest";
import { focusedDeconstructJsonSchema } from "@/lib/graph/focused-investigation";
// ── helpers ────────────────────────────────────────────────────────────── // ── helpers ──────────────────────────────────────────────────────────────
@@ -36,6 +37,14 @@ function makeMockProvider(inventedTargetNodeId) {
// ── Boundary test ──────────────────────────────────────────────────────── // ── Boundary test ────────────────────────────────────────────────────────
describe("focused-deconstruct targetNodeId identity boundary", () => { describe("focused-deconstruct targetNodeId identity boundary", () => {
beforeEach(() => {
vi.resetModules();
});
afterEach(() => {
vi.doUnmock("@/lib/llm/provider");
});
it("request targetNodeId overrides model-invented targetNodeId", async () => { it("request targetNodeId overrides model-invented targetNodeId", async () => {
const requestTargetNodeId = "nk04xvk"; // original graph node ID const requestTargetNodeId = "nk04xvk"; // original graph node ID
const inventedModelId = "invented-model-id"; const inventedModelId = "invented-model-id";
@@ -73,6 +82,45 @@ describe("focused-deconstruct targetNodeId identity boundary", () => {
expect(json.targetNodeId).not.toBe(inventedModelId); expect(json.targetNodeId).not.toBe(inventedModelId);
}); });
it("supplies the focused-deconstruction schema through the provider seam", async () => {
const generateReconstruction = vi.fn().mockResolvedValue({
response: {
targetNodeId: "model-id",
observations: [],
uncertainties: [],
assumptions: [],
relationships: [],
possibleFollowUpQuestions: [],
},
providerApiPath: "/api/chat",
providerExecution: { chatRequestAttempted: true },
});
vi.doMock("@/lib/llm/provider", () => ({
getProvider: () => ({ generateReconstruction }),
}));
const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js");
const response = await POST(new Request("http://localhost/api/focused-investigation/deconstruct", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
targetNodeId: "node-id",
targetLabel: "label",
targetDescription: "description",
centralStatement: "central statement",
question: "question?",
answer: "answer.",
}),
}));
expect(response.status).toBe(200);
expect(generateReconstruction).toHaveBeenCalledWith(
expect.any(String),
process.env.OLLAMA_MODEL,
focusedDeconstructJsonSchema,
);
});
it("semantic fields pass through unchanged from model", async () => { it("semantic fields pass through unchanged from model", async () => {
const mockObs = ["doc is minimal", "processes in founder's head"]; const mockObs = ["doc is minimal", "processes in founder's head"];
const mockUnc = ["whether formal docs can capture tacit knowledge"]; const mockUnc = ["whether formal docs can capture tacit knowledge"];
@@ -157,9 +205,6 @@ describe("focused-deconstruct targetNodeId identity boundary", () => {
}); });
it("full identity path: request → response → contribution", async () => { it("full identity path: request → response → contribution", async () => {
// Reset modules to avoid mock leakage from earlier tests
vi.resetModules();
const originalNodeId = "nk04xvk"; const originalNodeId = "nk04xvk";
const modelInventedId = "investigation_node_responsibility_distribution_autonomy"; const modelInventedId = "investigation_node_responsibility_distribution_autonomy";
@@ -231,8 +276,6 @@ describe("focused-deconstruct targetNodeId identity boundary", () => {
}); });
it("provider envelope fields do not leak into API response", async () => { it("provider envelope fields do not leak into API response", async () => {
vi.resetModules();
vi.doMock("@/lib/llm/provider", () => ({ vi.doMock("@/lib/llm/provider", () => ({
getProvider: () => ({ getProvider: () => ({
generateReconstruction: vi.fn().mockResolvedValue({ generateReconstruction: vi.fn().mockResolvedValue({
+26 -5
View File
@@ -5,6 +5,7 @@ import {
createOpenAIReconstructionProvider, createOpenAIReconstructionProvider,
getProvider, getProvider,
normaliseOpenAITransportResponse, normaliseOpenAITransportResponse,
reconstructionJsonSchema,
} from "@/lib/llm/provider.js"; } from "@/lib/llm/provider.js";
import { reconstructionV2Schema } from "@/lib/reconstruction/schema.js"; import { reconstructionV2Schema } from "@/lib/reconstruction/schema.js";
import { z } from "zod"; import { z } from "zod";
@@ -40,11 +41,7 @@ describe("OllamaLlmProvider chat capability detection", () => {
messages: [{ role: "user", content: "prompt" }], messages: [{ role: "user", content: "prompt" }],
stream: false, stream: false,
}); });
expect(chatRequest.format).toBeTypeOf("object"); expect(chatRequest.format).toEqual(reconstructionJsonSchema);
expect(chatRequest.format).not.toBe("json");
const formatText = JSON.stringify(chatRequest.format);
expect(formatText).toContain("relationship");
expect(formatText).toContain("evidenceType");
expect(result).toMatchObject({ expect(result).toMatchObject({
response: {}, response: {},
providerApiPath: "/api/chat", providerApiPath: "/api/chat",
@@ -63,6 +60,30 @@ describe("OllamaLlmProvider chat capability detection", () => {
} }
}); });
it("uses a supplied structured-output schema for the chat request", async () => {
const originalBaseUrl = process.env.OLLAMA_BASE_URL;
const outputSchema = {
type: "object",
properties: { focused: { type: "string" } },
required: ["focused"],
};
const fetchSpy = vi.fn()
.mockResolvedValueOnce({ ok: true, text: async () => "" })
.mockResolvedValueOnce({ ok: true, json: async () => ({ message: { content: "{}" } }) });
vi.stubGlobal("fetch", fetchSpy);
process.env.OLLAMA_BASE_URL = "http://ollama.test";
try {
__resetChatSupportForTests();
await getProvider().generateReconstruction("prompt", "configured-model", outputSchema);
expect(JSON.parse(fetchSpy.mock.calls[1][1].body).format).toEqual(outputSchema);
} finally {
vi.unstubAllGlobals();
if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
else process.env.OLLAMA_BASE_URL = originalBaseUrl;
}
});
it("reports chat-skipped generate fallback execution", async () => { it("reports chat-skipped generate fallback execution", async () => {
const originalBaseUrl = process.env.OLLAMA_BASE_URL; const originalBaseUrl = process.env.OLLAMA_BASE_URL;
const fetchSpy = vi.fn() const fetchSpy = vi.fn()