fix(confidence-engine): supply focused deconstruction schema
This commit is contained in:
@@ -1,5 +1,9 @@
|
|||||||
import { getProvider } from "@/lib/llm/provider";
|
import { getProvider } from "@/lib/llm/provider";
|
||||||
import { buildFocusedDeconstructPrompt, validateFocusedDeconstructSchema } from "@/lib/graph/focused-investigation";
|
import {
|
||||||
|
buildFocusedDeconstructPrompt,
|
||||||
|
focusedDeconstructJsonSchema,
|
||||||
|
validateFocusedDeconstructSchema,
|
||||||
|
} from "@/lib/graph/focused-investigation";
|
||||||
|
|
||||||
export async function POST(request) {
|
export async function POST(request) {
|
||||||
try {
|
try {
|
||||||
@@ -52,7 +56,11 @@ export async function POST(request) {
|
|||||||
|
|
||||||
const provider = getProvider();
|
const provider = getProvider();
|
||||||
const startedAt = Date.now();
|
const startedAt = Date.now();
|
||||||
const wrapper = await provider.generateReconstruction(prompt, process.env.OLLAMA_MODEL);
|
const wrapper = await provider.generateReconstruction(
|
||||||
|
prompt,
|
||||||
|
process.env.OLLAMA_MODEL,
|
||||||
|
focusedDeconstructJsonSchema,
|
||||||
|
);
|
||||||
const elapsedMs = Date.now() - startedAt;
|
const elapsedMs = Date.now() - startedAt;
|
||||||
|
|
||||||
// Unwrap the semantic deconstruction from the provider envelope.
|
// Unwrap the semantic deconstruction from the provider envelope.
|
||||||
|
|||||||
@@ -156,6 +156,13 @@
|
|||||||
- The first reconstruction-only Terra request reached inference successfully, but native-fetch parsing relied on SDK-only `output_text` and could not extract the raw response. The provider now reads documented `output[].content[].output_text` parts in response order while retaining the convenience-property path.
|
- The first reconstruction-only Terra request reached inference successfully, but native-fetch parsing relied on SDK-only `output_text` and could not extract the raw response. The provider now reads documented `output[].content[].output_text` parts in response order while retaining the convenience-property path.
|
||||||
- No tool-call assumption was introduced; canonical schema, prompt, transport compatibility, and reasoning remain unchanged. Zero live calls occurred during this correction; next boundary is exactly one live Terra reconstruction with no retry and no Ollama call.
|
- No tool-call assumption was introduced; canonical schema, prompt, transport compatibility, and reasoning remain unchanged. Zero live calls occurred during this correction; next boundary is exactly one live Terra reconstruction with no retry and no Ollama call.
|
||||||
|
|
||||||
|
## Focused-deconstruction structured-output transport
|
||||||
|
|
||||||
|
- Confirmed mismatch: the focused route requested and validated its six-field deconstruction contract while Ollama `/api/chat` was constrained to the initial reconstruction schema.
|
||||||
|
- The focused route now supplies `focusedDeconstructJsonSchema`; `generateReconstruction()` accepts it as an optional chat-format argument, while initial reconstruction callers retain the default `reconstructionJsonSchema`.
|
||||||
|
- Provider tests prove default and alternate schema transport. Focused route tests model the real provider wrapper, preserve `wrapper.response` unwrapping, and verify the schema argument without module/mock-state contamination.
|
||||||
|
- Previous failed live focused-deconstruction observations remain invalid semantic evidence. Zero live calls occurred during implementation and apparatus correction. Next boundary: exactly one substantive compound-answer live observation through the corrected production route.
|
||||||
|
|
||||||
## Current product architecture
|
## Current product architecture
|
||||||
|
|
||||||
Three distinct routes, not a single page:
|
Three distinct routes, not a single page:
|
||||||
|
|||||||
@@ -9,6 +9,20 @@ const FOCUSED_ANSWER_SCHEMA_FIELDS = [
|
|||||||
"possibleFollowUpQuestions",
|
"possibleFollowUpQuestions",
|
||||||
];
|
];
|
||||||
|
|
||||||
|
export const focusedDeconstructJsonSchema = {
|
||||||
|
type: "object",
|
||||||
|
properties: {
|
||||||
|
targetNodeId: { type: "string" },
|
||||||
|
observations: { type: "array", items: { type: "string" } },
|
||||||
|
uncertainties: { type: "array", items: { type: "string" } },
|
||||||
|
assumptions: { type: "array", items: { type: "string" } },
|
||||||
|
relationships: { type: "array", items: { type: "object" } },
|
||||||
|
possibleFollowUpQuestions: { type: "array", items: { type: "string" } },
|
||||||
|
},
|
||||||
|
required: FOCUSED_ANSWER_SCHEMA_FIELDS,
|
||||||
|
additionalProperties: false,
|
||||||
|
};
|
||||||
|
|
||||||
const FORBIDDEN_GRAPH_MUTATION_FIELDS = [
|
const FORBIDDEN_GRAPH_MUTATION_FIELDS = [
|
||||||
"addedNodes",
|
"addedNodes",
|
||||||
"updatedNodes",
|
"updatedNodes",
|
||||||
|
|||||||
+4
-4
@@ -96,7 +96,7 @@ function extractOpenAIResponseText(data) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let _chatSupported = null;
|
let _chatSupported = null;
|
||||||
const reconstructionJsonSchema = z.toJSONSchema(reconstructionV2Schema);
|
export const reconstructionJsonSchema = z.toJSONSchema(reconstructionV2Schema);
|
||||||
const openAIReconstructionJsonSchema = createOpenAIStrictSchema(
|
const openAIReconstructionJsonSchema = createOpenAIStrictSchema(
|
||||||
reconstructionJsonSchema,
|
reconstructionJsonSchema,
|
||||||
reconstructionV2Schema,
|
reconstructionV2Schema,
|
||||||
@@ -251,7 +251,7 @@ async function detectChatSupport(baseUrl, modelName) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
class OllamaLlmProvider {
|
class OllamaLlmProvider {
|
||||||
async generateReconstruction(scenario, modelName) {
|
async generateReconstruction(scenario, modelName, outputSchema) {
|
||||||
// scenario is ALREADY a fully-built prompt text (built by analyseScenario).
|
// scenario is ALREADY a fully-built prompt text (built by analyseScenario).
|
||||||
// Do NOT call buildPrompt() again — that would double-wrap the prompt.
|
// Do NOT call buildPrompt() again — that would double-wrap the prompt.
|
||||||
const prompt = scenario;
|
const prompt = scenario;
|
||||||
@@ -279,7 +279,7 @@ class OllamaLlmProvider {
|
|||||||
providerExecution.chatCapabilityDetected = chatSupported;
|
providerExecution.chatCapabilityDetected = chatSupported;
|
||||||
|
|
||||||
// ================================================================
|
// ================================================================
|
||||||
// Step 2: Try /api/chat if supported with the reconstruction schema
|
// Step 2: Try /api/chat if supported with the supplied or reconstruction schema
|
||||||
// ================================================================
|
// ================================================================
|
||||||
if (chatSupported) {
|
if (chatSupported) {
|
||||||
try {
|
try {
|
||||||
@@ -295,7 +295,7 @@ class OllamaLlmProvider {
|
|||||||
model: modelName,
|
model: modelName,
|
||||||
messages: [{ role: "user", content: prompt }],
|
messages: [{ role: "user", content: prompt }],
|
||||||
stream: false,
|
stream: false,
|
||||||
format: reconstructionJsonSchema,
|
format: outputSchema ?? reconstructionJsonSchema,
|
||||||
}),
|
}),
|
||||||
signal: controller.signal,
|
signal: controller.signal,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -6,7 +6,8 @@
|
|||||||
* API response targetNodeId regardless of what the model returns.
|
* API response targetNodeId regardless of what the model returns.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { describe, it, expect, vi } from "vitest";
|
import { afterEach, beforeEach, describe, it, expect, vi } from "vitest";
|
||||||
|
import { focusedDeconstructJsonSchema } from "@/lib/graph/focused-investigation";
|
||||||
|
|
||||||
// ── helpers ──────────────────────────────────────────────────────────────
|
// ── helpers ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -36,6 +37,14 @@ function makeMockProvider(inventedTargetNodeId) {
|
|||||||
// ── Boundary test ────────────────────────────────────────────────────────
|
// ── Boundary test ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
describe("focused-deconstruct targetNodeId identity boundary", () => {
|
describe("focused-deconstruct targetNodeId identity boundary", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.resetModules();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.doUnmock("@/lib/llm/provider");
|
||||||
|
});
|
||||||
|
|
||||||
it("request targetNodeId overrides model-invented targetNodeId", async () => {
|
it("request targetNodeId overrides model-invented targetNodeId", async () => {
|
||||||
const requestTargetNodeId = "nk04xvk"; // original graph node ID
|
const requestTargetNodeId = "nk04xvk"; // original graph node ID
|
||||||
const inventedModelId = "invented-model-id";
|
const inventedModelId = "invented-model-id";
|
||||||
@@ -73,6 +82,45 @@ describe("focused-deconstruct targetNodeId identity boundary", () => {
|
|||||||
expect(json.targetNodeId).not.toBe(inventedModelId);
|
expect(json.targetNodeId).not.toBe(inventedModelId);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("supplies the focused-deconstruction schema through the provider seam", async () => {
|
||||||
|
const generateReconstruction = vi.fn().mockResolvedValue({
|
||||||
|
response: {
|
||||||
|
targetNodeId: "model-id",
|
||||||
|
observations: [],
|
||||||
|
uncertainties: [],
|
||||||
|
assumptions: [],
|
||||||
|
relationships: [],
|
||||||
|
possibleFollowUpQuestions: [],
|
||||||
|
},
|
||||||
|
providerApiPath: "/api/chat",
|
||||||
|
providerExecution: { chatRequestAttempted: true },
|
||||||
|
});
|
||||||
|
vi.doMock("@/lib/llm/provider", () => ({
|
||||||
|
getProvider: () => ({ generateReconstruction }),
|
||||||
|
}));
|
||||||
|
const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js");
|
||||||
|
|
||||||
|
const response = await POST(new Request("http://localhost/api/focused-investigation/deconstruct", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({
|
||||||
|
targetNodeId: "node-id",
|
||||||
|
targetLabel: "label",
|
||||||
|
targetDescription: "description",
|
||||||
|
centralStatement: "central statement",
|
||||||
|
question: "question?",
|
||||||
|
answer: "answer.",
|
||||||
|
}),
|
||||||
|
}));
|
||||||
|
|
||||||
|
expect(response.status).toBe(200);
|
||||||
|
expect(generateReconstruction).toHaveBeenCalledWith(
|
||||||
|
expect.any(String),
|
||||||
|
process.env.OLLAMA_MODEL,
|
||||||
|
focusedDeconstructJsonSchema,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it("semantic fields pass through unchanged from model", async () => {
|
it("semantic fields pass through unchanged from model", async () => {
|
||||||
const mockObs = ["doc is minimal", "processes in founder's head"];
|
const mockObs = ["doc is minimal", "processes in founder's head"];
|
||||||
const mockUnc = ["whether formal docs can capture tacit knowledge"];
|
const mockUnc = ["whether formal docs can capture tacit knowledge"];
|
||||||
@@ -157,9 +205,6 @@ describe("focused-deconstruct targetNodeId identity boundary", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("full identity path: request → response → contribution", async () => {
|
it("full identity path: request → response → contribution", async () => {
|
||||||
// Reset modules to avoid mock leakage from earlier tests
|
|
||||||
vi.resetModules();
|
|
||||||
|
|
||||||
const originalNodeId = "nk04xvk";
|
const originalNodeId = "nk04xvk";
|
||||||
const modelInventedId = "investigation_node_responsibility_distribution_autonomy";
|
const modelInventedId = "investigation_node_responsibility_distribution_autonomy";
|
||||||
|
|
||||||
@@ -231,8 +276,6 @@ describe("focused-deconstruct targetNodeId identity boundary", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it("provider envelope fields do not leak into API response", async () => {
|
it("provider envelope fields do not leak into API response", async () => {
|
||||||
vi.resetModules();
|
|
||||||
|
|
||||||
vi.doMock("@/lib/llm/provider", () => ({
|
vi.doMock("@/lib/llm/provider", () => ({
|
||||||
getProvider: () => ({
|
getProvider: () => ({
|
||||||
generateReconstruction: vi.fn().mockResolvedValue({
|
generateReconstruction: vi.fn().mockResolvedValue({
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import {
|
|||||||
createOpenAIReconstructionProvider,
|
createOpenAIReconstructionProvider,
|
||||||
getProvider,
|
getProvider,
|
||||||
normaliseOpenAITransportResponse,
|
normaliseOpenAITransportResponse,
|
||||||
|
reconstructionJsonSchema,
|
||||||
} from "@/lib/llm/provider.js";
|
} from "@/lib/llm/provider.js";
|
||||||
import { reconstructionV2Schema } from "@/lib/reconstruction/schema.js";
|
import { reconstructionV2Schema } from "@/lib/reconstruction/schema.js";
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
@@ -40,11 +41,7 @@ describe("OllamaLlmProvider chat capability detection", () => {
|
|||||||
messages: [{ role: "user", content: "prompt" }],
|
messages: [{ role: "user", content: "prompt" }],
|
||||||
stream: false,
|
stream: false,
|
||||||
});
|
});
|
||||||
expect(chatRequest.format).toBeTypeOf("object");
|
expect(chatRequest.format).toEqual(reconstructionJsonSchema);
|
||||||
expect(chatRequest.format).not.toBe("json");
|
|
||||||
const formatText = JSON.stringify(chatRequest.format);
|
|
||||||
expect(formatText).toContain("relationship");
|
|
||||||
expect(formatText).toContain("evidenceType");
|
|
||||||
expect(result).toMatchObject({
|
expect(result).toMatchObject({
|
||||||
response: {},
|
response: {},
|
||||||
providerApiPath: "/api/chat",
|
providerApiPath: "/api/chat",
|
||||||
@@ -63,6 +60,30 @@ describe("OllamaLlmProvider chat capability detection", () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("uses a supplied structured-output schema for the chat request", async () => {
|
||||||
|
const originalBaseUrl = process.env.OLLAMA_BASE_URL;
|
||||||
|
const outputSchema = {
|
||||||
|
type: "object",
|
||||||
|
properties: { focused: { type: "string" } },
|
||||||
|
required: ["focused"],
|
||||||
|
};
|
||||||
|
const fetchSpy = vi.fn()
|
||||||
|
.mockResolvedValueOnce({ ok: true, text: async () => "" })
|
||||||
|
.mockResolvedValueOnce({ ok: true, json: async () => ({ message: { content: "{}" } }) });
|
||||||
|
vi.stubGlobal("fetch", fetchSpy);
|
||||||
|
process.env.OLLAMA_BASE_URL = "http://ollama.test";
|
||||||
|
|
||||||
|
try {
|
||||||
|
__resetChatSupportForTests();
|
||||||
|
await getProvider().generateReconstruction("prompt", "configured-model", outputSchema);
|
||||||
|
expect(JSON.parse(fetchSpy.mock.calls[1][1].body).format).toEqual(outputSchema);
|
||||||
|
} finally {
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
|
||||||
|
else process.env.OLLAMA_BASE_URL = originalBaseUrl;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it("reports chat-skipped generate fallback execution", async () => {
|
it("reports chat-skipped generate fallback execution", async () => {
|
||||||
const originalBaseUrl = process.env.OLLAMA_BASE_URL;
|
const originalBaseUrl = process.env.OLLAMA_BASE_URL;
|
||||||
const fetchSpy = vi.fn()
|
const fetchSpy = vi.fn()
|
||||||
|
|||||||
Reference in New Issue
Block a user