From 14630cf6b7e7c076484c8b3eee53a5b9520ce64a Mon Sep 17 00:00:00 2001 From: robbond Date: Mon, 7 Sep 2026 13:08:08 +0100 Subject: [PATCH] experiment(confidence-engine): trace focused deconstruction failures --- .../deconstruct/route.js | 60 ++++++++++++++++++- docs/current-handoff.md | 3 +- tests/focused-deconstruct-boundary.test.js | 60 +++++++++++++++++++ 3 files changed, 119 insertions(+), 4 deletions(-) diff --git a/app/api/focused-investigation/deconstruct/route.js b/app/api/focused-investigation/deconstruct/route.js index b84257c..2b34ea4 100644 --- a/app/api/focused-investigation/deconstruct/route.js +++ b/app/api/focused-investigation/deconstruct/route.js @@ -6,8 +6,11 @@ import { } from "@/lib/graph/focused-investigation"; export async function POST(request) { + let targetNodeId = null; + let startedAt = null; try { const body = await request.json(); + targetNodeId = body.targetNodeId ?? null; if (!body.targetNodeId || typeof body.targetNodeId !== "string") { return Response.json( @@ -55,20 +58,49 @@ export async function POST(request) { }); const provider = getProvider(); - const startedAt = Date.now(); + const modelName = getProviderModelName(); + startedAt = Date.now(); + console.info("[api/focused-investigation/deconstruct] start", { + targetNodeId, + modelName, + providerMode: process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER ?? "ollama", + startedAt, + }); const wrapper = await provider.generateReconstruction( prompt, - getProviderModelName(), + modelName, focusedDeconstructJsonSchema, ); const elapsedMs = Date.now() - startedAt; // Unwrap the semantic deconstruction from the provider envelope. const deconstruction = wrapper.response; + console.info("[api/focused-investigation/deconstruct] provider success", { + targetNodeId, + elapsedMs, + providerApiPath: wrapper.providerApiPath ?? null, + responsePresent: Boolean(deconstruction), + responseKeys: deconstruction && typeof deconstruction === "object" + ? Object.keys(deconstruction) + : [], + }); // Validate schema (required fields present, no graph-mutation fields) const validationErrors = validateFocusedDeconstructSchema(deconstruction); + console.info("[api/focused-investigation/deconstruct] validation", { + targetNodeId, + schemaValid: validationErrors.length === 0, + responseKeys: deconstruction && typeof deconstruction === "object" + ? Object.keys(deconstruction) + : [], + validationErrors, + }); if (validationErrors.length > 0) { + console.info("[api/focused-investigation/deconstruct] end", { + targetNodeId, + status: 502, + elapsedMs, + }); return Response.json( { success: false, @@ -81,7 +113,7 @@ export async function POST(request) { ); } - return Response.json({ + const response = Response.json({ success: true, targetNodeId: body.targetNodeId, observations: deconstruction.observations, @@ -91,7 +123,29 @@ export async function POST(request) { possibleFollowUpQuestions: deconstruction.possibleFollowUpQuestions, elapsedMs, }); + console.info("[api/focused-investigation/deconstruct] end", { + targetNodeId, + status: 200, + elapsedMs, + }); + return response; } catch (e) { + const elapsedMs = startedAt == null ? null : Date.now() - startedAt; + console.error("[api/focused-investigation/deconstruct] provider failure", { + targetNodeId, + elapsedMs, + errorName: e?.name ?? "Error", + errorMessage: e?.message ?? "Unknown server error", + statusCode: e?.statusCode ?? e?.status ?? null, + providerApiPath: e?.providerApiPath ?? null, + errorCode: e?.code ?? null, + errorParam: e?.param ?? null, + }); + console.info("[api/focused-investigation/deconstruct] end", { + targetNodeId, + status: 500, + elapsedMs, + }); return Response.json( { error: e.message || "Unknown server error" }, { status: 500 }, diff --git a/docs/current-handoff.md b/docs/current-handoff.md index 67854ea..357061b 100644 --- a/docs/current-handoff.md +++ b/docs/current-handoff.md @@ -39,7 +39,8 @@ If YES, the next live experiment is one timed/costed OpenAI UI investigation mea - The next live run exposed incomplete recursive strict projection: OpenAI rejected `relationships.items` because it lacked `additionalProperties: false`. The projector now recognizes every `type: "object"` node, including property-less objects in array items, and recursively enforces strict object schemas while preserving initial-reconstruction optionality/nullability behavior. - The latest Terra request then exposed an inconsistent root `properties`/`required` contract. The projector now derives `required` after projection from the surviving property keys, and recursive tests verify `properties`, `required`, and `additionalProperties` consistency. Property-less object strictness and initial-reconstruction transport behavior remain preserved. - Current deterministic final-fetch schema remains internally valid, yet the live rejection contradicts it. `CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA=1` now emits one safe, server-side structural summary immediately before the OpenAI fetch—no prompt, answer, request body, secret, or model output. -- Provider suite passes with zero live calls. Next boundary: one fresh focused UI submission with the OpenAI provider and schema-trace flags enabled; capture the single trace and OpenAI response, with no Retry. +- The focused-deconstruction route now emits complementary safe server diagnostics for start, provider success/failure, focused validation, and end status; the OpenAI schema trace remains provider-owned. No user content or secrets are logged. +- Focused route suite passes with zero live calls. Next boundary: one fresh focused UI submission with the OpenAI provider and schema-trace flags enabled; capture the single trace, route diagnostics, and OpenAI response, with no Retry. ## Repository checkpoint diff --git a/tests/focused-deconstruct-boundary.test.js b/tests/focused-deconstruct-boundary.test.js index 3168712..0f1028f 100644 --- a/tests/focused-deconstruct-boundary.test.js +++ b/tests/focused-deconstruct-boundary.test.js @@ -328,4 +328,64 @@ describe("focused-deconstruct targetNodeId identity boundary", () => { expect(json.providerApiPath).toBeUndefined(); expect(json.providerExecution).toBeUndefined(); }); + + it("preserves the 500 provider-failure contract while logging structural diagnostics", async () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + vi.doMock("@/lib/llm/provider", () => ({ + getProvider: () => ({ + generateReconstruction: vi.fn().mockRejectedValue(Object.assign(new Error("provider failed"), { + providerApiPath: "/v1/responses", + statusCode: 400, + })), + }), + getProviderModelName: () => "gpt-5.6-terra", + })); + const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js"); + + try { + const response = await POST(new Request("http://localhost/api/focused-investigation/deconstruct", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + targetNodeId: "node-id", targetLabel: "label", targetDescription: "description", + centralStatement: "central", question: "question?", answer: "answer.", + }), + })); + expect(response.status).toBe(500); + await expect(response.json()).resolves.toEqual({ error: "provider failed" }); + expect(errorSpy).toHaveBeenCalledWith( + "[api/focused-investigation/deconstruct] provider failure", + expect.objectContaining({ targetNodeId: "node-id", providerApiPath: "/v1/responses" }), + ); + } finally { + errorSpy.mockRestore(); + } + }); + + it("preserves the 502 validation-failure contract with diagnostics", async () => { + vi.doMock("@/lib/llm/provider", () => ({ + getProvider: () => ({ + generateReconstruction: vi.fn().mockResolvedValue({ + response: {}, providerApiPath: "/v1/responses", + }), + }), + getProviderModelName: () => "gpt-5.6-terra", + })); + const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js"); + const response = await POST(new Request("http://localhost/api/focused-investigation/deconstruct", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + targetNodeId: "node-id", targetLabel: "label", targetDescription: "description", + centralStatement: "central", question: "question?", answer: "answer.", + }), + })); + + expect(response.status).toBe(502); + await expect(response.json()).resolves.toMatchObject({ + success: false, + error: "Focused deconstruction result did not match expected schema", + targetNodeId: "node-id", + }); + }); });