fix(confidence-engine): expose reconstruction validation issues

This commit is contained in:
2026-09-05 13:48:48 +01:00
parent e5de8564a4
commit 84fc53f017
7 changed files with 115 additions and 5 deletions
+24 -2
View File
@@ -105,12 +105,23 @@ describe("app/api/cases/start route", () => {
});
it("returns provider/internal failures as 5xx without stack traces", async () => {
const rawResponse = `{"reconstruction":{"observedStates":[{"id":"obs-1"${"x".repeat(2500)}}]}}`;
mockStartCase.mockResolvedValue({
success: false,
error: "Provider unavailable",
diagnostics: { modelName: "llama3" },
statusCode: 502,
rawResponse: '{"reconstruction":{"summary":""}}',
analysisErrors: ["reconstruction: Required"],
validationIssues: [
{
path: ["reconstruction", "observedStates", 2, "description"],
code: "invalid_type",
message: "Required",
expected: "string",
received: "undefined",
},
],
rawResponse,
});
const { POST } = await import("@/app/api/cases/start/route.js");
@@ -125,7 +136,18 @@ describe("app/api/cases/start route", () => {
expect(response.status).toBe(502);
const body = await response.json();
expect(body).toHaveProperty("rawResponse");
expect(body.rawResponse).toBe('{"reconstruction":{"summary":""}}');
expect(body.rawResponse).toBe(rawResponse);
expect(body.rawResponse.length).toBeGreaterThan(2000);
expect(body.analysisErrors).toEqual(["reconstruction: Required"]);
expect(body.validationIssues).toEqual([
expect.objectContaining({
path: ["reconstruction", "observedStates", 2, "description"],
code: "invalid_type",
message: "Required",
expected: "string",
received: "undefined",
}),
]);
});
it("returns structured 500 on malformed JSON", async () => {
+30
View File
@@ -536,6 +536,36 @@ describe("lib/graph/orchestrator startCase", () => {
expect(result).toHaveProperty("rawResponse");
});
it("preserves structured reconstruction validation issues on analysis failure", async () => {
const validationIssues = [
{
path: ["reconstruction", "observedStates", 2, "description"],
code: "invalid_type",
message: "Required",
expected: "string",
received: "undefined",
},
];
mockAnalyseScenario.mockResolvedValue({
success: false,
error: "Scenario analysis failed",
errors: ["reconstruction: Required"],
validationIssues,
rawResponse: "x".repeat(2501),
});
const { startCase } = await import("@/lib/graph/orchestrator.js");
const result = await startCase({ scenario: "Scenario text" });
expect(result).toMatchObject({
success: false,
statusCode: 502,
analysisErrors: ["reconstruction: Required"],
validationIssues,
});
expect(result.rawResponse).toHaveLength(2501);
});
it("returns null selectedQuestion when neither analysis nor graph path yields a question", async () => {
mockAnalyseScenario.mockResolvedValue(
makeAnalysisResult({
@@ -192,6 +192,54 @@ describe("analyseScenario compatibility", () => {
expect(result.nextQuestion).toBeUndefined();
});
it("preserves nested validation issues and complete raw output on reconstruction failure", async () => {
mockGenerateReconstruction.mockResolvedValue({
inputClassification: {
primaryType: "unexplained_change",
secondaryTypes: [],
reasoningModes: [],
classificationReason: "reason",
confidence: "medium",
},
reconstruction: {
summary: "summary",
actors: [],
systemsOrObjects: [],
expectedStates: [],
observedStates: [{ id: "obs-1", confidence: "high" }],
differences: [],
knownTransitions: [],
unexplainedTransitions: [],
contradictions: [],
importantUnknowns: [],
plausibleInterpretations: [],
padding: "x".repeat(2500),
},
evidence: [],
nextQuestion: {
id: "q1",
question: "What changed?",
targets: [],
reason: "reason",
expectedInformationValue: "high",
},
});
const { analyseScenario } = await import("@/lib/analysis.js");
const result = await analyseScenario("Scenario text", {
promptVersion: "v0.3",
});
expect(result.success).toBe(false);
expect(result.rawResponse.length).toBeGreaterThan(2000);
expect(result.validationIssues).toContainEqual(expect.objectContaining({
path: ["reconstruction", "observedStates", 0, "description"],
code: "invalid_type",
message: "Required",
}));
expect(result.errors).toContain("reconstruction: Required");
});
it("succeeds when reported_claim is the only evidenceType mismatch", async () => {
mockGenerateReconstruction.mockResolvedValue({
inputClassification: {