diff --git a/lib/analysis.js b/lib/analysis.js index d214dca..3d964c1 100644 --- a/lib/analysis.js +++ b/lib/analysis.js @@ -5,7 +5,12 @@ import { getConfig } from "../lib/config.js"; import { getProvider } from "../lib/llm/provider.js"; -import { buildPrompt, PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION } from "../lib/reconstruction/prompt.js"; +import { + buildPrompt, + PROMPT_VERSIONS, + DEFAULT_PROMPT_VERSION, +} from "../lib/reconstruction/prompt.js"; +import { normaliseAnalysisResponse } from "../lib/reconstruction/compatibility.js"; import { reconstructionV2Schema, reconstructionSchema as reconstructionV1Schema, @@ -33,7 +38,10 @@ export async function analyseScenario(scenario, opts = {}) { return buildErrorResponse("Scenario cannot be empty", startTime); } if (trimmed.length > MAX_SCENARIO_LENGTH) { - return buildErrorResponse(`Scenario must be under ${MAX_SCENARIO_LENGTH} characters`, startTime); + return buildErrorResponse( + `Scenario must be under ${MAX_SCENARIO_LENGTH} characters`, + startTime, + ); } // ── Configuration check ──────────────────────────── @@ -50,18 +58,24 @@ export async function analyseScenario(scenario, opts = {}) { try { promptObj = await buildPrompt(trimmed, promptVersion); } catch (e) { - return buildErrorResponse(`Failed to build prompt: ${e.message}`, startTime); + return buildErrorResponse( + `Failed to build prompt: ${e.message}`, + startTime, + ); } // ── Call provider ────────────────────────────────── const provider = getProvider(); let rawResponse; try { - rawResponse = await provider.generateReconstruction(promptObj.prompt, OLLAMA_MODEL); + rawResponse = await provider.generateReconstruction( + promptObj.prompt, + OLLAMA_MODEL, + ); } catch (e) { return buildErrorResponse( e.message || "Provider error during analysis", - Date.now() - startTime + Date.now() - startTime, ); } @@ -75,16 +89,37 @@ export async function analyseScenario(scenario, opts = {}) { rawResponseStr = String(rawResponse).slice(0, 2000); } + const compatibility = normaliseAnalysisResponse(rawResponse); + const candidateResponse = compatibility.normalised; + // ── Validate against v0.2 schema (preferred) ────── - const resultV2 = tryValidateAgainstSchema(rawResponse, reconstructionV2Schema); + const resultV2 = tryValidateAgainstSchema( + candidateResponse, + reconstructionV2Schema, + ); if (resultV2.valid) { - return buildSuccessResultV2(resultV2.data, OLLAMA_MODEL, duration, promptVersion); + return buildSuccessResultV2( + resultV2.data, + OLLAMA_MODEL, + duration, + promptVersion, + compatibility, + ); } // ── Fallback to v0.1 schema ──────────────────────── - const resultV1 = tryValidateAgainstSchema(rawResponse, reconstructionV1Schema); + const resultV1 = tryValidateAgainstSchema( + candidateResponse, + reconstructionV1Schema, + ); if (resultV1.valid) { - return buildSuccessResultV1(resultV1.data, OLLAMA_MODEL, duration, promptVersion); + return buildSuccessResultV1( + resultV1.data, + OLLAMA_MODEL, + duration, + promptVersion, + compatibility, + ); } // ── Neither schema matched — partial failure ─────── @@ -93,17 +128,23 @@ export async function analyseScenario(scenario, opts = {}) { resultV2.error ?? resultV1.error, OLLAMA_MODEL, duration, - promptVersion + promptVersion, + compatibility, ); } /** Attempt validation against a Zod schema */ function tryValidateAgainstSchema(data, schema) { if (!schema.safeParse) { - return { valid: false, error: new Error("Schema does not support safeParse") }; + return { + valid: false, + error: new Error("Schema does not support safeParse"), + }; } const result = schema.safeParse(data); - return result.success ? { valid: true, data: result.data } : { valid: false, error: result.error }; + return result.success + ? { valid: true, data: result.data } + : { valid: false, error: result.error }; } // ── Result builders ────────────────────────────────── @@ -121,7 +162,15 @@ function buildErrorResponse(message, elapsed, statusCode = 500) { }; } -function buildSuccessResultV2(data, model, duration, version) { +function buildCompatibilityDiagnostics(compatibility) { + return { + compatibilityApplied: compatibility.changesApplied.length > 0, + compatibilityChanges: compatibility.changesApplied, + compatibilityWarnings: compatibility.warnings, + }; +} + +function buildSuccessResultV2(data, model, duration, version, compatibility) { return { success: true, validationStatus: "valid", @@ -134,10 +183,11 @@ function buildSuccessResultV2(data, model, duration, version) { evidence: data.evidence, nextQuestion: data.nextQuestion, errors: undefined, + ...buildCompatibilityDiagnostics(compatibility), }; } -function buildSuccessResultV1(data, model, duration, version) { +function buildSuccessResultV1(data, model, duration, version, compatibility) { return { success: true, validationStatus: "valid", @@ -150,14 +200,24 @@ function buildSuccessResultV1(data, model, duration, version) { evidence: undefined, nextQuestion: undefined, errors: undefined, + ...buildCompatibilityDiagnostics(compatibility), }; } -function buildPartialResult(rawResp, error, model, duration, version) { +function buildPartialResult( + rawResp, + error, + model, + duration, + version, + compatibility, +) { let errors = []; if (error && typeof error.flatten === "function") { errors = error.flatten().fieldErrors - ? Object.entries(error.flatten().fieldErrors).flatMap(([k, v]) => [`${k}: ${v.join(", ")}`]) + ? Object.entries(error.flatten().fieldErrors).flatMap(([k, v]) => [ + `${k}: ${v.join(", ")}`, + ]) : [String(error)]; } else if (error) { errors = [String(error).slice(0, 500)]; @@ -175,6 +235,7 @@ function buildPartialResult(rawResp, error, model, duration, version) { evidence: undefined, nextQuestion: undefined, errors, + ...buildCompatibilityDiagnostics(compatibility), }; } diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index abdb0aa..f041af1 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -34,6 +34,9 @@ function buildDiagnostics({ analysis, graph, graphReferenceValidation }) { nodeCount: graph?.nodes?.length ?? 0, edgeCount: graph?.edges?.length ?? 0, graphReferenceValidation, + compatibilityApplied: analysis?.compatibilityApplied ?? false, + compatibilityChanges: analysis?.compatibilityChanges ?? [], + compatibilityWarnings: analysis?.compatibilityWarnings ?? [], }; } diff --git a/lib/reconstruction/compatibility.js b/lib/reconstruction/compatibility.js new file mode 100644 index 0000000..efb5de2 --- /dev/null +++ b/lib/reconstruction/compatibility.js @@ -0,0 +1,40 @@ +function cloneJsonSafe(value) { + if (value == null) return value; + return JSON.parse(JSON.stringify(value)); +} + +export function normaliseAnalysisResponse(input) { + const normalised = cloneJsonSafe(input); + const changesApplied = []; + const warnings = []; + + if (!normalised || typeof normalised !== "object") { + return { normalised: input, changesApplied, warnings }; + } + + if (Array.isArray(normalised.evidence)) { + normalised.evidence = normalised.evidence.map((record, index) => { + if (!record || typeof record !== "object") return record; + + if (record.source === null) { + changesApplied.push({ + path: ["evidence", index, "source"], + change: "Converted null source to undefined", + }); + + const { source: _removed, ...rest } = record; + return rest; + } + + return record; + }); + } + + if (changesApplied.length > 0) { + warnings.push( + "Applied deterministic reconstruction compatibility normalisation", + ); + } + + return { normalised, changesApplied, warnings }; +} diff --git a/tests/graph/orchestrator.test.js b/tests/graph/orchestrator.test.js index a574821..682a270 100644 --- a/tests/graph/orchestrator.test.js +++ b/tests/graph/orchestrator.test.js @@ -46,6 +46,9 @@ function makeAnalysisResult(overrides = {}) { id: "q-1", question: "What denominator is being used for the complaint rate?", }, + compatibilityApplied: false, + compatibilityChanges: [], + compatibilityWarnings: [], ...overrides, }; } @@ -171,6 +174,29 @@ describe("lib/graph/orchestrator startCase", () => { expect(result.selectedQuestion).toBeNull(); }); + it("includes compatibility diagnostics when provided by analysis", async () => { + mockAnalyseScenario.mockResolvedValue( + makeAnalysisResult({ + compatibilityApplied: true, + compatibilityChanges: [ + { + path: ["evidence", 0, "source"], + change: "Converted null source to undefined", + }, + ], + compatibilityWarnings: [ + "Applied deterministic reconstruction compatibility normalisation", + ], + }), + ); + const { startCase } = await import("@/lib/graph/orchestrator.js"); + + const result = await startCase({ scenario: "Scenario text" }); + + expect(result.diagnostics.compatibilityApplied).toBe(true); + expect(result.diagnostics.compatibilityChanges).toHaveLength(1); + }); + it("exports placeholder updateCase", async () => { const { updateCase } = await import("@/lib/graph/orchestrator.js"); diff --git a/tests/reconstruction/compatibility.test.js b/tests/reconstruction/compatibility.test.js new file mode 100644 index 0000000..15fc5e5 --- /dev/null +++ b/tests/reconstruction/compatibility.test.js @@ -0,0 +1,179 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { normaliseAnalysisResponse } from "@/lib/reconstruction/compatibility.js"; + +const mockGenerateReconstruction = vi.fn(); + +vi.mock("@/lib/config.js", () => ({ + getConfig: () => ({ + ok: true, + config: { + OLLAMA_BASE_URL: "http://example.test", + OLLAMA_MODEL: "test-model", + }, + }), +})); + +vi.mock("@/lib/llm/provider.js", () => ({ + getProvider: () => ({ + generateReconstruction: (...args) => mockGenerateReconstruction(...args), + }), +})); + +vi.mock("@/lib/reconstruction/prompt.js", () => ({ + buildPrompt: async () => ({ prompt: "prompt", version: "v0.3" }), + PROMPT_VERSIONS: ["v0.1", "v0.2", "v0.3"], + DEFAULT_PROMPT_VERSION: "v0.3", +})); + +describe("normaliseAnalysisResponse", () => { + beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + }); + + it("leaves already-valid responses unchanged", () => { + const input = { + evidence: [ + { + id: "ev1", + description: "x", + evidenceType: "reported_statement", + confidence: "medium", + importance: "important", + source: "report", + }, + ], + }; + + const result = normaliseAnalysisResponse(input); + + expect(result.normalised).toEqual(input); + expect(result.changesApplied).toEqual([]); + }); + + it("normalises null evidence source deterministically", () => { + const input = { + evidence: [ + { + id: "ev1", + description: "x", + evidenceType: "reported_statement", + confidence: "medium", + importance: "important", + source: null, + }, + ], + }; + + const result = normaliseAnalysisResponse(input); + + expect(result.normalised.evidence[0]).not.toHaveProperty("source"); + expect(result.changesApplied).toHaveLength(1); + }); + + it("does not invent a next question", () => { + const input = { evidence: [] }; + const result = normaliseAnalysisResponse(input); + expect(result.normalised.nextQuestion).toBeUndefined(); + }); + + it("does not repair missing reasoning content", () => { + const input = { evidence: [{ source: null }] }; + const result = normaliseAnalysisResponse(input); + expect(result.normalised.reconstruction).toBeUndefined(); + }); +}); + +describe("analyseScenario compatibility", () => { + it("succeeds when the only mismatch is null evidence source", async () => { + mockGenerateReconstruction.mockResolvedValue({ + inputClassification: { + primaryType: "unexplained_change", + secondaryTypes: [], + reasoningModes: ["validate_measurement"], + classificationReason: "reason", + confidence: "medium", + }, + reconstruction: { + summary: "summary", + actors: [], + systemsOrObjects: [], + expectedStates: [], + observedStates: [], + differences: [], + knownTransitions: [], + unexplainedTransitions: [], + contradictions: [], + importantUnknowns: [], + plausibleInterpretations: [], + }, + evidence: [ + { + id: "ev1", + description: "desc", + evidenceType: "reported_statement", + source: null, + attribution: null, + confidence: "medium", + importance: "important", + }, + ], + nextQuestion: { + id: "q1", + question: "What denominator?", + targets: ["observedStates"], + reason: "reason", + expectedInformationValue: "high", + reasoningMode: "validate_measurement", + }, + }); + + const { analyseScenario } = await import("@/lib/analysis.js"); + const result = await analyseScenario("Scenario text", { + promptVersion: "v0.3", + }); + + expect(result.success).toBe(true); + expect(result.compatibilityApplied).toBe(true); + expect(result.compatibilityChanges).toHaveLength(1); + expect(result.evidence[0]).not.toHaveProperty("source"); + expect(result.nextQuestion.question).toBe("What denominator?"); + }); + + it("still fails when required reasoning content is missing", async () => { + mockGenerateReconstruction.mockResolvedValue({ + evidence: [ + { + id: "ev1", + description: "desc", + evidenceType: "reported_statement", + source: null, + attribution: null, + confidence: "medium", + importance: "important", + }, + ], + }); + + const { analyseScenario } = await import("@/lib/analysis.js"); + const result = await analyseScenario("Scenario text", { + promptVersion: "v0.3", + }); + + expect(result.success).toBe(false); + expect(result.compatibilityApplied).toBe(true); + expect(result.nextQuestion).toBeUndefined(); + }); + + it("malformed JSON still fails", async () => { + mockGenerateReconstruction.mockResolvedValue("{not valid json"); + + const { analyseScenario } = await import("@/lib/analysis.js"); + const result = await analyseScenario("Scenario text", { + promptVersion: "v0.3", + }); + + expect(result.success).toBe(false); + expect(result.compatibilityApplied).toBe(false); + }); +});