fix(confidence-engine): sanitize reasoning error responses
This commit is contained in:
@@ -0,0 +1,205 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
// ── Mock domain seam and provider at module level ────
|
||||
|
||||
const mockAnalyseScenario = vi.fn();
|
||||
|
||||
vi.mock("@/lib/analysis", () => ({
|
||||
analyseScenario: (...args) => mockAnalyseScenario(...args),
|
||||
PROMPT_VERSIONS: ["v1"],
|
||||
DEFAULT_PROMPT_VERSION: "v1",
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/llm/provider.js", () => ({
|
||||
getProvider: () => ({}),
|
||||
getProviderModelName: () => "gpt-5.6-terra",
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/supabase/api-auth.js", () => ({
|
||||
withAuthenticatedApi: (handler) => handler,
|
||||
}));
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────
|
||||
|
||||
function makeValidScenario() {
|
||||
return "The supplier changed delivery schedules without notice, causing our production line to halt.";
|
||||
}
|
||||
|
||||
function makeRequest(body) {
|
||||
return new Request("http://localhost/api/analyse", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
}
|
||||
|
||||
// ── Route tests — valid POST ────────────────────────
|
||||
|
||||
describe("POST /api/analyse — success contract", () => {
|
||||
it("returns structured analysis on success", async () => {
|
||||
mockAnalyseScenario.mockResolvedValue({
|
||||
success: true,
|
||||
inputClassification: "manufacturing",
|
||||
reconstruction: { summary: "Validated reconstruction summary" },
|
||||
evidence: [],
|
||||
nextQuestion: null,
|
||||
modelName: "gpt-5.6-terra",
|
||||
responseDurationMs: 1200,
|
||||
validationStatus: "passed",
|
||||
promptVersion: "v1",
|
||||
});
|
||||
|
||||
const { POST } = await import("@/app/api/analyse/route.js");
|
||||
const res = await POST(makeRequest({ scenario: makeValidScenario() }));
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
const data = await res.json();
|
||||
expect(typeof data.inputClassification).toBe("string");
|
||||
expect(data.reconstruction.summary).toBe("Validated reconstruction summary");
|
||||
});
|
||||
});
|
||||
|
||||
// ── Route tests — request validation ────────────────
|
||||
|
||||
describe("POST /api/analyse — request validation", () => {
|
||||
it("missing scenario → 400", async () => {
|
||||
const { POST } = await import("@/app/api/analyse/route.js");
|
||||
const res = await POST(makeRequest({}));
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
const data = await res.json();
|
||||
expect(data).toEqual({ error: "Request must include a 'scenario' string field" });
|
||||
});
|
||||
|
||||
it("null scenario → 400", async () => {
|
||||
const { POST } = await import("@/app/api/analyse/route.js");
|
||||
const res = await POST(makeRequest({ scenario: null }));
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
|
||||
it("non-string scenario → 400", async () => {
|
||||
const { POST } = await import("@/app/api/analyse/route.js");
|
||||
const res = await POST(makeRequest({ scenario: 123 }));
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Route tests — error handling ────────────────────
|
||||
|
||||
describe("POST /api/analyse — error boundary", () => {
|
||||
it("domain seam throws PROVIDER_UNAVAILABLE → sanitized 503 with generic message", async () => {
|
||||
const err = Object.assign(
|
||||
new Error("Ollama /api/generate request timed out after 5 minutes"),
|
||||
{ code: "PROVIDER_UNAVAILABLE", providerApiPath: "/api/generate" },
|
||||
);
|
||||
mockAnalyseScenario.mockImplementationOnce(async () => { throw err; });
|
||||
|
||||
const { POST } = await import("@/app/api/analyse/route.js");
|
||||
const res = await POST(makeRequest({ scenario: makeValidScenario() }));
|
||||
|
||||
expect(res.status).toBe(503);
|
||||
const data = await res.json();
|
||||
expect(data.error).toBe("Reasoning service is temporarily unavailable.");
|
||||
});
|
||||
|
||||
it("domain seam throws with diagnostics → sanitized response, preserved status", async () => {
|
||||
const rawResponse = `{"reconstruction":{"observedStates":[{"id":"obs-1"${"x".repeat(2500)}}]}}`;
|
||||
const err = Object.assign(
|
||||
new Error("Provider unavailable"),
|
||||
{
|
||||
statusCode: 502,
|
||||
providerApiPath: "/v1/responses",
|
||||
providerExecution: { chatRequestAttempted: true },
|
||||
rawResponse,
|
||||
},
|
||||
);
|
||||
mockAnalyseScenario.mockImplementationOnce(async () => { throw err; });
|
||||
|
||||
const { POST } = await import("@/app/api/analyse/route.js");
|
||||
const res = await POST(makeRequest({ scenario: makeValidScenario() }));
|
||||
|
||||
expect(res.status).toBe(502);
|
||||
const data = await res.json();
|
||||
expect(data.error).toBe("Reasoning request could not be completed.");
|
||||
expect(JSON.stringify(data)).not.toMatch(/provider unavailable|generate|llama3|rawResponse/i);
|
||||
});
|
||||
|
||||
it("domain seam throws without statusCode → 500", async () => {
|
||||
mockAnalyseScenario.mockImplementationOnce(async () => { throw new Error("unknown error"); });
|
||||
|
||||
const { POST } = await import("@/app/api/analyse/route.js");
|
||||
const res = await POST(makeRequest({ scenario: makeValidScenario() }));
|
||||
|
||||
expect(res.status).toBe(500);
|
||||
const data = await res.json();
|
||||
expect(data.error).toBe("Reasoning request could not be completed.");
|
||||
});
|
||||
|
||||
it("domain seam throws validation failure → preserved status", async () => {
|
||||
const err = Object.assign(new Error("Invalid input"), { statusCode: 400 });
|
||||
mockAnalyseScenario.mockImplementationOnce(async () => { throw err; });
|
||||
|
||||
const { POST } = await import("@/app/api/analyse/route.js");
|
||||
const res = await POST(makeRequest({ scenario: makeValidScenario() }));
|
||||
|
||||
expect(res.status).toBe(400);
|
||||
const data = await res.json();
|
||||
expect(data.error).toBe("Reasoning request could not be completed.");
|
||||
});
|
||||
|
||||
it("domain seam returns failure result with diagnostics → sanitized response", async () => {
|
||||
mockAnalyseScenario.mockResolvedValue({
|
||||
success: false,
|
||||
error: "Provider unavailable",
|
||||
diagnostics: { modelName: "llama3" },
|
||||
statusCode: 502,
|
||||
analysisErrors: ["reconstruction: Required"],
|
||||
validationIssues: [{ path: ["reconstruction"], code: "invalid_type", message: "Required" }],
|
||||
providerApiPath: "/api/generate",
|
||||
providerExecution: { generateRequestAttempted: true },
|
||||
rawResponse: '{"observedStates":[{"id":"x"}]}',
|
||||
});
|
||||
|
||||
const { POST } = await import("@/app/api/analyse/route.js");
|
||||
const res = await POST(makeRequest({ scenario: makeValidScenario() }));
|
||||
|
||||
expect(res.status).toBe(502);
|
||||
const data = await res.json();
|
||||
expect(data).toEqual({ error: "Reasoning request could not be completed." });
|
||||
expect(JSON.stringify(data)).not.toMatch(/provider unavailable|generate|llama3|rawResponse/i);
|
||||
});
|
||||
|
||||
it("domain seam throws without exposing raw provider/internal diagnostics", async () => {
|
||||
const err = Object.assign(
|
||||
new Error("Internal connection reset by peer — host=10.0.0.5:8080 key=sk-abc"),
|
||||
{ statusCode: 502, providerApiPath: "/internal/chat", providerExecution: { chatRequestAttempted: true } },
|
||||
);
|
||||
mockAnalyseScenario.mockImplementationOnce(async () => { throw err; });
|
||||
|
||||
const { POST } = await import("@/app/api/analyse/route.js");
|
||||
const res = await POST(makeRequest({ scenario: makeValidScenario() }));
|
||||
|
||||
expect(res.status).toBe(502);
|
||||
const data = await res.json();
|
||||
expect(JSON.stringify(data)).not.toMatch(/10\.0\.0\.5|8080|sk-abc|providerApiPath|providerExecution|Internal connection reset/i);
|
||||
});
|
||||
|
||||
it("domain seam returns failed analysis object → not spread into response", async () => {
|
||||
mockAnalyseScenario.mockResolvedValue({
|
||||
success: false,
|
||||
error: "Analysis failed",
|
||||
statusCode: 500,
|
||||
failedAnalysisObject: { raw: true, internal: "diagnostics" },
|
||||
});
|
||||
|
||||
const { POST } = await import("@/app/api/analyse/route.js");
|
||||
const res = await POST(makeRequest({ scenario: makeValidScenario() }));
|
||||
|
||||
expect(res.status).toBe(500);
|
||||
const data = await res.json();
|
||||
expect(data).toEqual({ error: "Reasoning request could not be completed." });
|
||||
expect(JSON.stringify(data)).not.toMatch(/failedAnalysisObject|internal|diagnostics/i);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user