experiment(confidence-engine): trace focused deconstruction failures
This commit is contained in:
@@ -6,8 +6,11 @@ import {
|
|||||||
} from "@/lib/graph/focused-investigation";
|
} from "@/lib/graph/focused-investigation";
|
||||||
|
|
||||||
export async function POST(request) {
|
export async function POST(request) {
|
||||||
|
let targetNodeId = null;
|
||||||
|
let startedAt = null;
|
||||||
try {
|
try {
|
||||||
const body = await request.json();
|
const body = await request.json();
|
||||||
|
targetNodeId = body.targetNodeId ?? null;
|
||||||
|
|
||||||
if (!body.targetNodeId || typeof body.targetNodeId !== "string") {
|
if (!body.targetNodeId || typeof body.targetNodeId !== "string") {
|
||||||
return Response.json(
|
return Response.json(
|
||||||
@@ -55,20 +58,49 @@ export async function POST(request) {
|
|||||||
});
|
});
|
||||||
|
|
||||||
const provider = getProvider();
|
const provider = getProvider();
|
||||||
const startedAt = Date.now();
|
const modelName = getProviderModelName();
|
||||||
|
startedAt = Date.now();
|
||||||
|
console.info("[api/focused-investigation/deconstruct] start", {
|
||||||
|
targetNodeId,
|
||||||
|
modelName,
|
||||||
|
providerMode: process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER ?? "ollama",
|
||||||
|
startedAt,
|
||||||
|
});
|
||||||
const wrapper = await provider.generateReconstruction(
|
const wrapper = await provider.generateReconstruction(
|
||||||
prompt,
|
prompt,
|
||||||
getProviderModelName(),
|
modelName,
|
||||||
focusedDeconstructJsonSchema,
|
focusedDeconstructJsonSchema,
|
||||||
);
|
);
|
||||||
const elapsedMs = Date.now() - startedAt;
|
const elapsedMs = Date.now() - startedAt;
|
||||||
|
|
||||||
// Unwrap the semantic deconstruction from the provider envelope.
|
// Unwrap the semantic deconstruction from the provider envelope.
|
||||||
const deconstruction = wrapper.response;
|
const deconstruction = wrapper.response;
|
||||||
|
console.info("[api/focused-investigation/deconstruct] provider success", {
|
||||||
|
targetNodeId,
|
||||||
|
elapsedMs,
|
||||||
|
providerApiPath: wrapper.providerApiPath ?? null,
|
||||||
|
responsePresent: Boolean(deconstruction),
|
||||||
|
responseKeys: deconstruction && typeof deconstruction === "object"
|
||||||
|
? Object.keys(deconstruction)
|
||||||
|
: [],
|
||||||
|
});
|
||||||
|
|
||||||
// Validate schema (required fields present, no graph-mutation fields)
|
// Validate schema (required fields present, no graph-mutation fields)
|
||||||
const validationErrors = validateFocusedDeconstructSchema(deconstruction);
|
const validationErrors = validateFocusedDeconstructSchema(deconstruction);
|
||||||
|
console.info("[api/focused-investigation/deconstruct] validation", {
|
||||||
|
targetNodeId,
|
||||||
|
schemaValid: validationErrors.length === 0,
|
||||||
|
responseKeys: deconstruction && typeof deconstruction === "object"
|
||||||
|
? Object.keys(deconstruction)
|
||||||
|
: [],
|
||||||
|
validationErrors,
|
||||||
|
});
|
||||||
if (validationErrors.length > 0) {
|
if (validationErrors.length > 0) {
|
||||||
|
console.info("[api/focused-investigation/deconstruct] end", {
|
||||||
|
targetNodeId,
|
||||||
|
status: 502,
|
||||||
|
elapsedMs,
|
||||||
|
});
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{
|
{
|
||||||
success: false,
|
success: false,
|
||||||
@@ -81,7 +113,7 @@ export async function POST(request) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
return Response.json({
|
const response = Response.json({
|
||||||
success: true,
|
success: true,
|
||||||
targetNodeId: body.targetNodeId,
|
targetNodeId: body.targetNodeId,
|
||||||
observations: deconstruction.observations,
|
observations: deconstruction.observations,
|
||||||
@@ -91,7 +123,29 @@ export async function POST(request) {
|
|||||||
possibleFollowUpQuestions: deconstruction.possibleFollowUpQuestions,
|
possibleFollowUpQuestions: deconstruction.possibleFollowUpQuestions,
|
||||||
elapsedMs,
|
elapsedMs,
|
||||||
});
|
});
|
||||||
|
console.info("[api/focused-investigation/deconstruct] end", {
|
||||||
|
targetNodeId,
|
||||||
|
status: 200,
|
||||||
|
elapsedMs,
|
||||||
|
});
|
||||||
|
return response;
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
const elapsedMs = startedAt == null ? null : Date.now() - startedAt;
|
||||||
|
console.error("[api/focused-investigation/deconstruct] provider failure", {
|
||||||
|
targetNodeId,
|
||||||
|
elapsedMs,
|
||||||
|
errorName: e?.name ?? "Error",
|
||||||
|
errorMessage: e?.message ?? "Unknown server error",
|
||||||
|
statusCode: e?.statusCode ?? e?.status ?? null,
|
||||||
|
providerApiPath: e?.providerApiPath ?? null,
|
||||||
|
errorCode: e?.code ?? null,
|
||||||
|
errorParam: e?.param ?? null,
|
||||||
|
});
|
||||||
|
console.info("[api/focused-investigation/deconstruct] end", {
|
||||||
|
targetNodeId,
|
||||||
|
status: 500,
|
||||||
|
elapsedMs,
|
||||||
|
});
|
||||||
return Response.json(
|
return Response.json(
|
||||||
{ error: e.message || "Unknown server error" },
|
{ error: e.message || "Unknown server error" },
|
||||||
{ status: 500 },
|
{ status: 500 },
|
||||||
|
|||||||
@@ -39,7 +39,8 @@ If YES, the next live experiment is one timed/costed OpenAI UI investigation mea
|
|||||||
- The next live run exposed incomplete recursive strict projection: OpenAI rejected `relationships.items` because it lacked `additionalProperties: false`. The projector now recognizes every `type: "object"` node, including property-less objects in array items, and recursively enforces strict object schemas while preserving initial-reconstruction optionality/nullability behavior.
|
- The next live run exposed incomplete recursive strict projection: OpenAI rejected `relationships.items` because it lacked `additionalProperties: false`. The projector now recognizes every `type: "object"` node, including property-less objects in array items, and recursively enforces strict object schemas while preserving initial-reconstruction optionality/nullability behavior.
|
||||||
- The latest Terra request then exposed an inconsistent root `properties`/`required` contract. The projector now derives `required` after projection from the surviving property keys, and recursive tests verify `properties`, `required`, and `additionalProperties` consistency. Property-less object strictness and initial-reconstruction transport behavior remain preserved.
|
- The latest Terra request then exposed an inconsistent root `properties`/`required` contract. The projector now derives `required` after projection from the surviving property keys, and recursive tests verify `properties`, `required`, and `additionalProperties` consistency. Property-less object strictness and initial-reconstruction transport behavior remain preserved.
|
||||||
- Current deterministic final-fetch schema remains internally valid, yet the live rejection contradicts it. `CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA=1` now emits one safe, server-side structural summary immediately before the OpenAI fetch—no prompt, answer, request body, secret, or model output.
|
- Current deterministic final-fetch schema remains internally valid, yet the live rejection contradicts it. `CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA=1` now emits one safe, server-side structural summary immediately before the OpenAI fetch—no prompt, answer, request body, secret, or model output.
|
||||||
- Provider suite passes with zero live calls. Next boundary: one fresh focused UI submission with the OpenAI provider and schema-trace flags enabled; capture the single trace and OpenAI response, with no Retry.
|
- The focused-deconstruction route now emits complementary safe server diagnostics for start, provider success/failure, focused validation, and end status; the OpenAI schema trace remains provider-owned. No user content or secrets are logged.
|
||||||
|
- Focused route suite passes with zero live calls. Next boundary: one fresh focused UI submission with the OpenAI provider and schema-trace flags enabled; capture the single trace, route diagnostics, and OpenAI response, with no Retry.
|
||||||
|
|
||||||
## Repository checkpoint
|
## Repository checkpoint
|
||||||
|
|
||||||
|
|||||||
@@ -328,4 +328,64 @@ describe("focused-deconstruct targetNodeId identity boundary", () => {
|
|||||||
expect(json.providerApiPath).toBeUndefined();
|
expect(json.providerApiPath).toBeUndefined();
|
||||||
expect(json.providerExecution).toBeUndefined();
|
expect(json.providerExecution).toBeUndefined();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("preserves the 500 provider-failure contract while logging structural diagnostics", async () => {
|
||||||
|
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||||
|
vi.doMock("@/lib/llm/provider", () => ({
|
||||||
|
getProvider: () => ({
|
||||||
|
generateReconstruction: vi.fn().mockRejectedValue(Object.assign(new Error("provider failed"), {
|
||||||
|
providerApiPath: "/v1/responses",
|
||||||
|
statusCode: 400,
|
||||||
|
})),
|
||||||
|
}),
|
||||||
|
getProviderModelName: () => "gpt-5.6-terra",
|
||||||
|
}));
|
||||||
|
const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js");
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await POST(new Request("http://localhost/api/focused-investigation/deconstruct", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({
|
||||||
|
targetNodeId: "node-id", targetLabel: "label", targetDescription: "description",
|
||||||
|
centralStatement: "central", question: "question?", answer: "answer.",
|
||||||
|
}),
|
||||||
|
}));
|
||||||
|
expect(response.status).toBe(500);
|
||||||
|
await expect(response.json()).resolves.toEqual({ error: "provider failed" });
|
||||||
|
expect(errorSpy).toHaveBeenCalledWith(
|
||||||
|
"[api/focused-investigation/deconstruct] provider failure",
|
||||||
|
expect.objectContaining({ targetNodeId: "node-id", providerApiPath: "/v1/responses" }),
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
errorSpy.mockRestore();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("preserves the 502 validation-failure contract with diagnostics", async () => {
|
||||||
|
vi.doMock("@/lib/llm/provider", () => ({
|
||||||
|
getProvider: () => ({
|
||||||
|
generateReconstruction: vi.fn().mockResolvedValue({
|
||||||
|
response: {}, providerApiPath: "/v1/responses",
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
getProviderModelName: () => "gpt-5.6-terra",
|
||||||
|
}));
|
||||||
|
const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js");
|
||||||
|
const response = await POST(new Request("http://localhost/api/focused-investigation/deconstruct", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({
|
||||||
|
targetNodeId: "node-id", targetLabel: "label", targetDescription: "description",
|
||||||
|
centralStatement: "central", question: "question?", answer: "answer.",
|
||||||
|
}),
|
||||||
|
}));
|
||||||
|
|
||||||
|
expect(response.status).toBe(502);
|
||||||
|
await expect(response.json()).resolves.toMatchObject({
|
||||||
|
success: false,
|
||||||
|
error: "Focused deconstruction result did not match expected schema",
|
||||||
|
targetNodeId: "node-id",
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user