fix(confidence-engine): sanitize reasoning error responses

This commit is contained in:
2026-09-10 11:39:13 +01:00
parent b9d49ff277
commit 2cb2d556fd
11 changed files with 304 additions and 44 deletions
+19 -3
View File
@@ -26,7 +26,7 @@ async function post(request) {
if (!result.success) {
return Response.json(
{ ...result, reconstruction: result.reconstruction || null },
{ error: "Reasoning request could not be completed." },
{ status: Number(result.statusCode) || 500 },
);
}
@@ -42,9 +42,25 @@ async function post(request) {
promptVersion: result.promptVersion,
});
} catch (e) {
if (e.code === "PROVIDER_UNAVAILABLE") {
return Response.json(
{ error: "Reasoning service is temporarily unavailable." },
{ status: 503 },
);
}
const validationFailed = e?.validationFailed || e?.code === "VALIDATION_FAILED";
if (validationFailed) {
return Response.json(
{ success: false, error: "Reasoning request could not be completed." },
{ status: Number(e.statusCode) || 500 },
);
}
const statusCode = Number(e.statusCode) || 500;
return Response.json(
{ error: e.message || "Unknown server error", responseDurationMs: 0 },
{ status: 500 },
{ error: "Reasoning request could not be completed." },
{ status: statusCode },
);
}
}
+3 -1
View File
@@ -55,7 +55,9 @@ async function post(request) {
{
success: false,
stage: error.statusCode === 400 ? "request_validation" : "provider",
error: error.message ?? "Overview synthesis failed",
error: error.statusCode === 400
? "Invalid overview request"
: "Reasoning request could not be completed.",
},
{ status: error.statusCode }
);
+2 -8
View File
@@ -48,14 +48,8 @@ async function post(request) {
return Response.json(
{
success: false,
error: diagnostics.error,
validationErrors: result.validationErrors,
diagnostics: result.diagnostics,
analysisErrors: result.analysisErrors,
validationIssues: result.validationIssues,
providerApiPath: result.providerApiPath,
providerExecution: result.providerExecution,
rawResponse: result.rawResponse ?? undefined,
error: status === 400 ? diagnostics.error : "Reasoning request could not be completed.",
...(status === 400 ? { validationErrors: result.validationErrors } : {}),
},
{ status },
);
+3 -1
View File
@@ -57,7 +57,9 @@ async function post(request) {
{
success: false,
stage: error.statusCode === 400 ? "request_validation" : "provider",
error: error.message ?? "Synthesis failed",
error: error.statusCode === 400
? "Invalid synthesis request"
: "Reasoning request could not be completed.",
},
{ status: error.statusCode }
);
@@ -154,7 +154,7 @@ async function post(request) {
);
}
return Response.json(
{ error: e.message || "Unknown server error" },
{ error: "Reasoning request could not be completed." },
{ status: 500 },
);
}
+7
View File
@@ -16,6 +16,13 @@
- The existing generic Retry UX remains unchanged. A live deployed outage had already proven investigation preservation.
- `/api/cases/start` and `/api/cases/update` outage sanitization remain separately unverified; this change does not claim those routes are fixed.
## Reasoning API browser error boundary hardened
- All reasoning HTTP error responses now sanitize raw provider diagnostics: `e.message` / raw internals never leak to the client.
- Verified on: `/api/cases/start`, `/api/focused-investigation/deconstruct`, `/api/cases/overview`, `/api/cases/synthesis`, `/api/analyse`.
- Provider/internal diagnostics remain server-side only.
- No reasoning semantics changed.
## v0.62d production Docker packaging — LIVE PROVEN
- `Dockerfile` — minimal multi-stage Alpine build (Node 22), Next.js standalone output mode
+205
View File
@@ -0,0 +1,205 @@
import { describe, expect, it, vi } from "vitest";
// ── Mock domain seam and provider at module level ────
const mockAnalyseScenario = vi.fn();
vi.mock("@/lib/analysis", () => ({
analyseScenario: (...args) => mockAnalyseScenario(...args),
PROMPT_VERSIONS: ["v1"],
DEFAULT_PROMPT_VERSION: "v1",
}));
vi.mock("@/lib/llm/provider.js", () => ({
getProvider: () => ({}),
getProviderModelName: () => "gpt-5.6-terra",
}));
vi.mock("@/lib/supabase/api-auth.js", () => ({
withAuthenticatedApi: (handler) => handler,
}));
// ── Helpers ─────────────────────────────────────────
function makeValidScenario() {
return "The supplier changed delivery schedules without notice, causing our production line to halt.";
}
function makeRequest(body) {
return new Request("http://localhost/api/analyse", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
});
}
// ── Route tests — valid POST ────────────────────────
describe("POST /api/analyse — success contract", () => {
it("returns structured analysis on success", async () => {
mockAnalyseScenario.mockResolvedValue({
success: true,
inputClassification: "manufacturing",
reconstruction: { summary: "Validated reconstruction summary" },
evidence: [],
nextQuestion: null,
modelName: "gpt-5.6-terra",
responseDurationMs: 1200,
validationStatus: "passed",
promptVersion: "v1",
});
const { POST } = await import("@/app/api/analyse/route.js");
const res = await POST(makeRequest({ scenario: makeValidScenario() }));
expect(res.status).toBe(200);
const data = await res.json();
expect(typeof data.inputClassification).toBe("string");
expect(data.reconstruction.summary).toBe("Validated reconstruction summary");
});
});
// ── Route tests — request validation ────────────────
describe("POST /api/analyse — request validation", () => {
it("missing scenario → 400", async () => {
const { POST } = await import("@/app/api/analyse/route.js");
const res = await POST(makeRequest({}));
expect(res.status).toBe(400);
const data = await res.json();
expect(data).toEqual({ error: "Request must include a 'scenario' string field" });
});
it("null scenario → 400", async () => {
const { POST } = await import("@/app/api/analyse/route.js");
const res = await POST(makeRequest({ scenario: null }));
expect(res.status).toBe(400);
});
it("non-string scenario → 400", async () => {
const { POST } = await import("@/app/api/analyse/route.js");
const res = await POST(makeRequest({ scenario: 123 }));
expect(res.status).toBe(400);
});
});
// ── Route tests — error handling ────────────────────
describe("POST /api/analyse — error boundary", () => {
it("domain seam throws PROVIDER_UNAVAILABLE → sanitized 503 with generic message", async () => {
const err = Object.assign(
new Error("Ollama /api/generate request timed out after 5 minutes"),
{ code: "PROVIDER_UNAVAILABLE", providerApiPath: "/api/generate" },
);
mockAnalyseScenario.mockImplementationOnce(async () => { throw err; });
const { POST } = await import("@/app/api/analyse/route.js");
const res = await POST(makeRequest({ scenario: makeValidScenario() }));
expect(res.status).toBe(503);
const data = await res.json();
expect(data.error).toBe("Reasoning service is temporarily unavailable.");
});
it("domain seam throws with diagnostics → sanitized response, preserved status", async () => {
const rawResponse = `{"reconstruction":{"observedStates":[{"id":"obs-1"${"x".repeat(2500)}}]}}`;
const err = Object.assign(
new Error("Provider unavailable"),
{
statusCode: 502,
providerApiPath: "/v1/responses",
providerExecution: { chatRequestAttempted: true },
rawResponse,
},
);
mockAnalyseScenario.mockImplementationOnce(async () => { throw err; });
const { POST } = await import("@/app/api/analyse/route.js");
const res = await POST(makeRequest({ scenario: makeValidScenario() }));
expect(res.status).toBe(502);
const data = await res.json();
expect(data.error).toBe("Reasoning request could not be completed.");
expect(JSON.stringify(data)).not.toMatch(/provider unavailable|generate|llama3|rawResponse/i);
});
it("domain seam throws without statusCode → 500", async () => {
mockAnalyseScenario.mockImplementationOnce(async () => { throw new Error("unknown error"); });
const { POST } = await import("@/app/api/analyse/route.js");
const res = await POST(makeRequest({ scenario: makeValidScenario() }));
expect(res.status).toBe(500);
const data = await res.json();
expect(data.error).toBe("Reasoning request could not be completed.");
});
it("domain seam throws validation failure → preserved status", async () => {
const err = Object.assign(new Error("Invalid input"), { statusCode: 400 });
mockAnalyseScenario.mockImplementationOnce(async () => { throw err; });
const { POST } = await import("@/app/api/analyse/route.js");
const res = await POST(makeRequest({ scenario: makeValidScenario() }));
expect(res.status).toBe(400);
const data = await res.json();
expect(data.error).toBe("Reasoning request could not be completed.");
});
it("domain seam returns failure result with diagnostics → sanitized response", async () => {
mockAnalyseScenario.mockResolvedValue({
success: false,
error: "Provider unavailable",
diagnostics: { modelName: "llama3" },
statusCode: 502,
analysisErrors: ["reconstruction: Required"],
validationIssues: [{ path: ["reconstruction"], code: "invalid_type", message: "Required" }],
providerApiPath: "/api/generate",
providerExecution: { generateRequestAttempted: true },
rawResponse: '{"observedStates":[{"id":"x"}]}',
});
const { POST } = await import("@/app/api/analyse/route.js");
const res = await POST(makeRequest({ scenario: makeValidScenario() }));
expect(res.status).toBe(502);
const data = await res.json();
expect(data).toEqual({ error: "Reasoning request could not be completed." });
expect(JSON.stringify(data)).not.toMatch(/provider unavailable|generate|llama3|rawResponse/i);
});
it("domain seam throws without exposing raw provider/internal diagnostics", async () => {
const err = Object.assign(
new Error("Internal connection reset by peer — host=10.0.0.5:8080 key=sk-abc"),
{ statusCode: 502, providerApiPath: "/internal/chat", providerExecution: { chatRequestAttempted: true } },
);
mockAnalyseScenario.mockImplementationOnce(async () => { throw err; });
const { POST } = await import("@/app/api/analyse/route.js");
const res = await POST(makeRequest({ scenario: makeValidScenario() }));
expect(res.status).toBe(502);
const data = await res.json();
expect(JSON.stringify(data)).not.toMatch(/10\.0\.0\.5|8080|sk-abc|providerApiPath|providerExecution|Internal connection reset/i);
});
it("domain seam returns failed analysis object → not spread into response", async () => {
mockAnalyseScenario.mockResolvedValue({
success: false,
error: "Analysis failed",
statusCode: 500,
failedAnalysisObject: { raw: true, internal: "diagnostics" },
});
const { POST } = await import("@/app/api/analyse/route.js");
const res = await POST(makeRequest({ scenario: makeValidScenario() }));
expect(res.status).toBe(500);
const data = await res.json();
expect(data).toEqual({ error: "Reasoning request could not be completed." });
expect(JSON.stringify(data)).not.toMatch(/failedAnalysisObject|internal|diagnostics/i);
});
});
+37 -1
View File
@@ -1,4 +1,4 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { describe, expect, it, vi } from "vitest";
const mockSynthesize = vi.fn();
@@ -11,6 +11,10 @@ vi.mock("@/lib/llm/provider.js", () => ({
getProviderModelName: () => "gpt-5.6-terra",
}));
vi.mock("@/lib/supabase/api-auth.js", () => ({
withAuthenticatedApi: (handler) => handler,
}));
describe("POST /api/cases/overview provider routing", () => {
beforeEach(() => mockSynthesize.mockClear());
@@ -26,4 +30,36 @@ describe("POST /api/cases/overview provider routing", () => {
expect(response.status).toBe(200);
expect(mockSynthesize.mock.calls[0][1]).toMatchObject({ modelName: "gpt-5.6-terra" });
});
it("sanitizes provider failure details", async () => {
const error = Object.assign(
new Error("Ollama /api/generate returned 500 from private host"),
{ statusCode: 502 },
);
mockSynthesize.mockImplementationOnce(async () => {
throw error;
});
const { POST } = await import("@/app/api/cases/overview/route.js");
const response = await POST(new Request("http://localhost/api/cases/overview", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ situationGraph: { nodes: [], edges: [] } }),
}));
expect(response.status).toBe(502);
const rawData = await response.text();
const data = JSON.parse(rawData);
expect(data.success).toBe(false);
expect(data.stage).toBe("provider");
expect(data).toEqual({
success: false,
stage: "provider",
error: "Reasoning request could not be completed.",
});
// Prove raw provider diagnostics are NOT exposed at the route boundary
expect(rawData).not.toContain(error.message);
});
});
+3 -21
View File
@@ -169,7 +169,7 @@ describe("app/api/cases/start route", () => {
expect(JSON.stringify(body)).not.toMatch(/ollama|generate|timed out/i);
});
it("returns provider/internal failures as 5xx without stack traces", async () => {
it("sanitizes provider/internal failures at the browser boundary", async () => {
const rawResponse = `{"reconstruction":{"observedStates":[{"id":"obs-1"${"x".repeat(2500)}}]}}`;
mockStartCase.mockResolvedValue({
success: false,
@@ -207,26 +207,8 @@ describe("app/api/cases/start route", () => {
expect(response.status).toBe(502);
const body = await response.json();
expect(body).toHaveProperty("rawResponse");
expect(body.rawResponse).toBe(rawResponse);
expect(body.rawResponse.length).toBeGreaterThan(2000);
expect(body.analysisErrors).toEqual(["reconstruction: Required"]);
expect(body.validationIssues).toEqual([
expect.objectContaining({
path: ["reconstruction", "observedStates", 2, "description"],
code: "invalid_type",
message: "Required",
expected: "string",
received: "undefined",
}),
]);
expect(body.providerApiPath).toBe("/api/generate");
expect(body.providerExecution).toEqual({
chatCapabilityDetected: false,
chatRequestAttempted: false,
chatRequestSucceeded: false,
generateRequestAttempted: true,
});
expect(body).toEqual({ success: false, error: "Reasoning request could not be completed." });
expect(JSON.stringify(body)).not.toMatch(/provider unavailable|generate|llama3|rawResponse/i);
expect(errorSpy).toHaveBeenCalledTimes(1);
expect(errorSpy).toHaveBeenCalledWith(
"[api/cases/start] error response",
@@ -13,6 +13,10 @@ vi.mock("@/lib/llm/provider.js", () => ({
getProviderModelName: () => "gpt-5.6-terra",
}));
vi.mock("@/lib/supabase/api-auth.js", () => ({
withAuthenticatedApi: (handler) => handler,
}));
// ── Helpers ─────────────────────────────────────────────────
function makeValidGraph() {
@@ -110,10 +114,13 @@ describe("POST /api/cases/synthesis — error cases", () => {
});
it("domain seam throws with statusCode → mapped status", async () => {
mockSynthesize.mockRejectedValue(new Error("Provider failed"));
// Add statusCode property to the error object after creation
const err = Object.assign(new Error("Provider failed"), { statusCode: 502 });
mockSynthesize.mockRejectedValue(err);
const err = Object.assign(
new Error("Ollama /api/generate returned 500 from private host"),
{ statusCode: 502 },
);
mockSynthesize.mockImplementationOnce(async () => {
throw err;
});
const { POST } = await import("@/app/api/cases/synthesis/route.js");
const res = await POST(makeRequest({ situationGraph: makeValidGraph() }));
@@ -122,10 +129,17 @@ describe("POST /api/cases/synthesis — error cases", () => {
const data = await res.json();
expect(data.success).toBe(false);
expect(data.stage).toBe("provider");
expect(data).toEqual({
success: false,
stage: "provider",
error: "Reasoning request could not be completed.",
});
});
it("domain seam throws without statusCode → 500", async () => {
mockSynthesize.mockRejectedValue(new Error("unknown error"));
mockSynthesize.mockImplementationOnce(async () => {
throw new Error("unknown error");
});
const { POST } = await import("@/app/api/cases/synthesis/route.js");
const res = await POST(makeRequest({ situationGraph: makeValidGraph() }));
@@ -138,7 +152,9 @@ describe("POST /api/cases/synthesis — error cases", () => {
it("domain seam throws 400 → mapped to 400", async () => {
const err = Object.assign(new Error("Invalid input"), { statusCode: 400 });
mockSynthesize.mockRejectedValue(err);
mockSynthesize.mockImplementationOnce(async () => {
throw err;
});
const { POST } = await import("@/app/api/cases/synthesis/route.js");
const res = await POST(makeRequest({ situationGraph: makeValidGraph() }));
+2 -2
View File
@@ -360,7 +360,7 @@ describe("focused-deconstruct targetNodeId identity boundary", () => {
expect(json.providerExecution).toBeUndefined();
});
it("preserves the 500 provider-failure contract while logging structural diagnostics", async () => {
it("sanitizes generic provider failures while logging structural diagnostics", async () => {
const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
vi.doMock("@/lib/llm/provider", () => ({
getProvider: () => ({
@@ -383,7 +383,7 @@ describe("focused-deconstruct targetNodeId identity boundary", () => {
}),
}));
expect(response.status).toBe(500);
await expect(response.json()).resolves.toEqual({ error: "provider failed" });
await expect(response.json()).resolves.toEqual({ error: "Reasoning request could not be completed." });
expect(errorSpy).toHaveBeenCalledWith(
"[api/focused-investigation/deconstruct] provider failure",
expect.objectContaining({ targetNodeId: "node-id", providerApiPath: "/v1/responses" }),