fix(confidence-engine): expose reconstruction validation issues
This commit is contained in:
@@ -105,12 +105,23 @@ describe("app/api/cases/start route", () => {
|
||||
});
|
||||
|
||||
it("returns provider/internal failures as 5xx without stack traces", async () => {
|
||||
const rawResponse = `{"reconstruction":{"observedStates":[{"id":"obs-1"${"x".repeat(2500)}}]}}`;
|
||||
mockStartCase.mockResolvedValue({
|
||||
success: false,
|
||||
error: "Provider unavailable",
|
||||
diagnostics: { modelName: "llama3" },
|
||||
statusCode: 502,
|
||||
rawResponse: '{"reconstruction":{"summary":""}}',
|
||||
analysisErrors: ["reconstruction: Required"],
|
||||
validationIssues: [
|
||||
{
|
||||
path: ["reconstruction", "observedStates", 2, "description"],
|
||||
code: "invalid_type",
|
||||
message: "Required",
|
||||
expected: "string",
|
||||
received: "undefined",
|
||||
},
|
||||
],
|
||||
rawResponse,
|
||||
});
|
||||
|
||||
const { POST } = await import("@/app/api/cases/start/route.js");
|
||||
@@ -125,7 +136,18 @@ describe("app/api/cases/start route", () => {
|
||||
expect(response.status).toBe(502);
|
||||
const body = await response.json();
|
||||
expect(body).toHaveProperty("rawResponse");
|
||||
expect(body.rawResponse).toBe('{"reconstruction":{"summary":""}}');
|
||||
expect(body.rawResponse).toBe(rawResponse);
|
||||
expect(body.rawResponse.length).toBeGreaterThan(2000);
|
||||
expect(body.analysisErrors).toEqual(["reconstruction: Required"]);
|
||||
expect(body.validationIssues).toEqual([
|
||||
expect.objectContaining({
|
||||
path: ["reconstruction", "observedStates", 2, "description"],
|
||||
code: "invalid_type",
|
||||
message: "Required",
|
||||
expected: "string",
|
||||
received: "undefined",
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("returns structured 500 on malformed JSON", async () => {
|
||||
|
||||
@@ -536,6 +536,36 @@ describe("lib/graph/orchestrator startCase", () => {
|
||||
expect(result).toHaveProperty("rawResponse");
|
||||
});
|
||||
|
||||
it("preserves structured reconstruction validation issues on analysis failure", async () => {
|
||||
const validationIssues = [
|
||||
{
|
||||
path: ["reconstruction", "observedStates", 2, "description"],
|
||||
code: "invalid_type",
|
||||
message: "Required",
|
||||
expected: "string",
|
||||
received: "undefined",
|
||||
},
|
||||
];
|
||||
mockAnalyseScenario.mockResolvedValue({
|
||||
success: false,
|
||||
error: "Scenario analysis failed",
|
||||
errors: ["reconstruction: Required"],
|
||||
validationIssues,
|
||||
rawResponse: "x".repeat(2501),
|
||||
});
|
||||
const { startCase } = await import("@/lib/graph/orchestrator.js");
|
||||
|
||||
const result = await startCase({ scenario: "Scenario text" });
|
||||
|
||||
expect(result).toMatchObject({
|
||||
success: false,
|
||||
statusCode: 502,
|
||||
analysisErrors: ["reconstruction: Required"],
|
||||
validationIssues,
|
||||
});
|
||||
expect(result.rawResponse).toHaveLength(2501);
|
||||
});
|
||||
|
||||
it("returns null selectedQuestion when neither analysis nor graph path yields a question", async () => {
|
||||
mockAnalyseScenario.mockResolvedValue(
|
||||
makeAnalysisResult({
|
||||
|
||||
@@ -192,6 +192,54 @@ describe("analyseScenario compatibility", () => {
|
||||
expect(result.nextQuestion).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves nested validation issues and complete raw output on reconstruction failure", async () => {
|
||||
mockGenerateReconstruction.mockResolvedValue({
|
||||
inputClassification: {
|
||||
primaryType: "unexplained_change",
|
||||
secondaryTypes: [],
|
||||
reasoningModes: [],
|
||||
classificationReason: "reason",
|
||||
confidence: "medium",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: "summary",
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [{ id: "obs-1", confidence: "high" }],
|
||||
differences: [],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: [],
|
||||
importantUnknowns: [],
|
||||
plausibleInterpretations: [],
|
||||
padding: "x".repeat(2500),
|
||||
},
|
||||
evidence: [],
|
||||
nextQuestion: {
|
||||
id: "q1",
|
||||
question: "What changed?",
|
||||
targets: [],
|
||||
reason: "reason",
|
||||
expectedInformationValue: "high",
|
||||
},
|
||||
});
|
||||
|
||||
const { analyseScenario } = await import("@/lib/analysis.js");
|
||||
const result = await analyseScenario("Scenario text", {
|
||||
promptVersion: "v0.3",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.rawResponse.length).toBeGreaterThan(2000);
|
||||
expect(result.validationIssues).toContainEqual(expect.objectContaining({
|
||||
path: ["reconstruction", "observedStates", 0, "description"],
|
||||
code: "invalid_type",
|
||||
message: "Required",
|
||||
}));
|
||||
expect(result.errors).toContain("reconstruction: Required");
|
||||
});
|
||||
|
||||
it("succeeds when reported_claim is the only evidenceType mismatch", async () => {
|
||||
mockGenerateReconstruction.mockResolvedValue({
|
||||
inputClassification: {
|
||||
|
||||
Reference in New Issue
Block a user