robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(confidence-engine): supply focused deconstruction schema

Browse Source
This commit is contained in:
robbond
2026-09-06 16:21:26 +01:00
parent 188dd04ab9
commit a93b6798cc
6 changed files with 110 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/focused-investigation/deconstruct/route.js
+10 -2
View File
@@ -1,5 +1,9 @@
import { getProvider } from "@/lib/llm/provider";
import { buildFocusedDeconstructPrompt, validateFocusedDeconstructSchema } from "@/lib/graph/focused-investigation";
import {
buildFocusedDeconstructPrompt,
focusedDeconstructJsonSchema,
validateFocusedDeconstructSchema,
} from "@/lib/graph/focused-investigation";
export async function POST(request) {
try {
@@ -52,7 +56,11 @@ export async function POST(request) {
const provider = getProvider();
const startedAt = Date.now();
const wrapper = await provider.generateReconstruction(prompt, process.env.OLLAMA_MODEL);
const wrapper = await provider.generateReconstruction(
prompt,
process.env.OLLAMA_MODEL,
focusedDeconstructJsonSchema,
);
const elapsedMs = Date.now() - startedAt;
// Unwrap the semantic deconstruction from the provider envelope.
docs/current-handoff.md
+7
View File
@@ -156,6 +156,13 @@
- The first reconstruction-only Terra request reached inference successfully, but native-fetch parsing relied on SDK-only `output_text` and could not extract the raw response. The provider now reads documented `output[].content[].output_text` parts in response order while retaining the convenience-property path.
- No tool-call assumption was introduced; canonical schema, prompt, transport compatibility, and reasoning remain unchanged. Zero live calls occurred during this correction; next boundary is exactly one live Terra reconstruction with no retry and no Ollama call.
## Focused-deconstruction structured-output transport
- Confirmed mismatch: the focused route requested and validated its six-field deconstruction contract while Ollama `/api/chat` was constrained to the initial reconstruction schema.
- The focused route now supplies `focusedDeconstructJsonSchema`; `generateReconstruction()` accepts it as an optional chat-format argument, while initial reconstruction callers retain the default `reconstructionJsonSchema`.
- Provider tests prove default and alternate schema transport. Focused route tests model the real provider wrapper, preserve `wrapper.response` unwrapping, and verify the schema argument without module/mock-state contamination.
- Previous failed live focused-deconstruction observations remain invalid semantic evidence. Zero live calls occurred during implementation and apparatus correction. Next boundary: exactly one substantive compound-answer live observation through the corrected production route.
## Current product architecture
Three distinct routes, not a single page:
lib/graph/focused-investigation.js
+14
View File
@@ -9,6 +9,20 @@ const FOCUSED_ANSWER_SCHEMA_FIELDS = [
"possibleFollowUpQuestions",
];
export const focusedDeconstructJsonSchema = {
type: "object",
properties: {
targetNodeId: { type: "string" },
observations: { type: "array", items: { type: "string" } },
uncertainties: { type: "array", items: { type: "string" } },
assumptions: { type: "array", items: { type: "string" } },
relationships: { type: "array", items: { type: "object" } },
possibleFollowUpQuestions: { type: "array", items: { type: "string" } },
},
required: FOCUSED_ANSWER_SCHEMA_FIELDS,
additionalProperties: false,
};
const FORBIDDEN_GRAPH_MUTATION_FIELDS = [
"addedNodes",
"updatedNodes",
lib/llm/provider.js
+4 -4
View File
@@ -96,7 +96,7 @@ function extractOpenAIResponseText(data) {
}
let _chatSupported = null;
const reconstructionJsonSchema = z.toJSONSchema(reconstructionV2Schema);
export const reconstructionJsonSchema = z.toJSONSchema(reconstructionV2Schema);
const openAIReconstructionJsonSchema = createOpenAIStrictSchema(
reconstructionJsonSchema,
reconstructionV2Schema,
@@ -251,7 +251,7 @@ async function detectChatSupport(baseUrl, modelName) {
}
class OllamaLlmProvider {
async generateReconstruction(scenario, modelName) {
async generateReconstruction(scenario, modelName, outputSchema) {
// scenario is ALREADY a fully-built prompt text (built by analyseScenario).
// Do NOT call buildPrompt() again — that would double-wrap the prompt.
const prompt = scenario;
@@ -279,7 +279,7 @@ class OllamaLlmProvider {
providerExecution.chatCapabilityDetected = chatSupported;
// ================================================================
// Step 2: Try /api/chat if supported with the reconstruction schema
// Step 2: Try /api/chat if supported with the supplied or reconstruction schema
// ================================================================
if (chatSupported) {
try {
@@ -295,7 +295,7 @@ class OllamaLlmProvider {
model: modelName,
messages: [{ role: "user", content: prompt }],
stream: false,
format: reconstructionJsonSchema,
format: outputSchema ?? reconstructionJsonSchema,
}),
signal: controller.signal,
});
tests/focused-deconstruct-boundary.test.js
+49 -6
View File
@@ -6,7 +6,8 @@
* API response targetNodeId regardless of what the model returns.
*/
import { describe, it, expect, vi } from "vitest";
import { afterEach, beforeEach, describe, it, expect, vi } from "vitest";
import { focusedDeconstructJsonSchema } from "@/lib/graph/focused-investigation";
// ── helpers ──────────────────────────────────────────────────────────────
@@ -36,6 +37,14 @@ function makeMockProvider(inventedTargetNodeId) {
// ── Boundary test ────────────────────────────────────────────────────────
describe("focused-deconstruct targetNodeId identity boundary", () => {
beforeEach(() => {
vi.resetModules();
});
afterEach(() => {
vi.doUnmock("@/lib/llm/provider");
});
it("request targetNodeId overrides model-invented targetNodeId", async () => {
const requestTargetNodeId = "nk04xvk"; // original graph node ID
const inventedModelId = "invented-model-id";
@@ -73,6 +82,45 @@ describe("focused-deconstruct targetNodeId identity boundary", () => {
expect(json.targetNodeId).not.toBe(inventedModelId);
});
it("supplies the focused-deconstruction schema through the provider seam", async () => {
const generateReconstruction = vi.fn().mockResolvedValue({
response: {
targetNodeId: "model-id",
observations: [],
uncertainties: [],
assumptions: [],
relationships: [],
possibleFollowUpQuestions: [],
},
providerApiPath: "/api/chat",
providerExecution: { chatRequestAttempted: true },
});
vi.doMock("@/lib/llm/provider", () => ({
getProvider: () => ({ generateReconstruction }),
}));
const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js");
const response = await POST(new Request("http://localhost/api/focused-investigation/deconstruct", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
targetNodeId: "node-id",
targetLabel: "label",
targetDescription: "description",
centralStatement: "central statement",
question: "question?",
answer: "answer.",
}),
}));
expect(response.status).toBe(200);
expect(generateReconstruction).toHaveBeenCalledWith(
expect.any(String),
process.env.OLLAMA_MODEL,
focusedDeconstructJsonSchema,
);
});
it("semantic fields pass through unchanged from model", async () => {
const mockObs = ["doc is minimal", "processes in founder's head"];
const mockUnc = ["whether formal docs can capture tacit knowledge"];
@@ -157,9 +205,6 @@ describe("focused-deconstruct targetNodeId identity boundary", () => {
});
it("full identity path: request → response → contribution", async () => {
// Reset modules to avoid mock leakage from earlier tests
vi.resetModules();
const originalNodeId = "nk04xvk";
const modelInventedId = "investigation_node_responsibility_distribution_autonomy";
@@ -231,8 +276,6 @@ describe("focused-deconstruct targetNodeId identity boundary", () => {
});
it("provider envelope fields do not leak into API response", async () => {
vi.resetModules();
vi.doMock("@/lib/llm/provider", () => ({
getProvider: () => ({
generateReconstruction: vi.fn().mockResolvedValue({
tests/llm/provider.test.js
+26 -5
View File
@@ -5,6 +5,7 @@ import {
createOpenAIReconstructionProvider,
getProvider,
normaliseOpenAITransportResponse,
reconstructionJsonSchema,
} from "@/lib/llm/provider.js";
import { reconstructionV2Schema } from "@/lib/reconstruction/schema.js";
import { z } from "zod";
@@ -40,11 +41,7 @@ describe("OllamaLlmProvider chat capability detection", () => {
messages: [{ role: "user", content: "prompt" }],
stream: false,
});
expect(chatRequest.format).toBeTypeOf("object");
expect(chatRequest.format).not.toBe("json");
const formatText = JSON.stringify(chatRequest.format);
expect(formatText).toContain("relationship");
expect(formatText).toContain("evidenceType");
expect(chatRequest.format).toEqual(reconstructionJsonSchema);
expect(result).toMatchObject({
response: {},
providerApiPath: "/api/chat",
@@ -63,6 +60,30 @@ describe("OllamaLlmProvider chat capability detection", () => {
}
});
it("uses a supplied structured-output schema for the chat request", async () => {
const originalBaseUrl = process.env.OLLAMA_BASE_URL;
const outputSchema = {
type: "object",
properties: { focused: { type: "string" } },
required: ["focused"],
};
const fetchSpy = vi.fn()
.mockResolvedValueOnce({ ok: true, text: async () => "" })
.mockResolvedValueOnce({ ok: true, json: async () => ({ message: { content: "{}" } }) });
vi.stubGlobal("fetch", fetchSpy);
process.env.OLLAMA_BASE_URL = "http://ollama.test";
try {
__resetChatSupportForTests();
await getProvider().generateReconstruction("prompt", "configured-model", outputSchema);
expect(JSON.parse(fetchSpy.mock.calls[1][1].body).format).toEqual(outputSchema);
} finally {
vi.unstubAllGlobals();
if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
else process.env.OLLAMA_BASE_URL = originalBaseUrl;
}
});
it("reports chat-skipped generate fallback execution", async () => {
const originalBaseUrl = process.env.OLLAMA_BASE_URL;
const fetchSpy = vi.fn()