robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: normalise compatible live reconstruction responses

Browse Source
This commit is contained in:
robbond
2026-08-02 07:50:02 +01:00
parent 575b8fd971
commit 02a6ecd0da
5 changed files with 325 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/analysis.js
+77 -16
View File
@@ -5,7 +5,12 @@
import { getConfig } from "../lib/config.js";
import { getProvider } from "../lib/llm/provider.js";
import { buildPrompt, PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION } from "../lib/reconstruction/prompt.js";
import {
buildPrompt,
PROMPT_VERSIONS,
DEFAULT_PROMPT_VERSION,
} from "../lib/reconstruction/prompt.js";
import { normaliseAnalysisResponse } from "../lib/reconstruction/compatibility.js";
import {
reconstructionV2Schema,
reconstructionSchema as reconstructionV1Schema,
@@ -33,7 +38,10 @@ export async function analyseScenario(scenario, opts = {}) {
return buildErrorResponse("Scenario cannot be empty", startTime);
}
if (trimmed.length > MAX_SCENARIO_LENGTH) {
return buildErrorResponse(`Scenario must be under ${MAX_SCENARIO_LENGTH} characters`, startTime);
return buildErrorResponse(
`Scenario must be under ${MAX_SCENARIO_LENGTH} characters`,
startTime,
);
}
// ── Configuration check ────────────────────────────
@@ -50,18 +58,24 @@ export async function analyseScenario(scenario, opts = {}) {
try {
promptObj = await buildPrompt(trimmed, promptVersion);
} catch (e) {
return buildErrorResponse(`Failed to build prompt: ${e.message}`, startTime);
return buildErrorResponse(
`Failed to build prompt: ${e.message}`,
startTime,
);
}
// ── Call provider ──────────────────────────────────
const provider = getProvider();
let rawResponse;
try {
rawResponse = await provider.generateReconstruction(promptObj.prompt, OLLAMA_MODEL);
rawResponse = await provider.generateReconstruction(
promptObj.prompt,
OLLAMA_MODEL,
);
} catch (e) {
return buildErrorResponse(
e.message || "Provider error during analysis",
Date.now() - startTime
Date.now() - startTime,
);
}
@@ -75,16 +89,37 @@ export async function analyseScenario(scenario, opts = {}) {
rawResponseStr = String(rawResponse).slice(0, 2000);
}
const compatibility = normaliseAnalysisResponse(rawResponse);
const candidateResponse = compatibility.normalised;
// ── Validate against v0.2 schema (preferred) ──────
const resultV2 = tryValidateAgainstSchema(rawResponse, reconstructionV2Schema);
const resultV2 = tryValidateAgainstSchema(
candidateResponse,
reconstructionV2Schema,
);
if (resultV2.valid) {
return buildSuccessResultV2(resultV2.data, OLLAMA_MODEL, duration, promptVersion);
return buildSuccessResultV2(
resultV2.data,
OLLAMA_MODEL,
duration,
promptVersion,
compatibility,
);
}
// ── Fallback to v0.1 schema ────────────────────────
const resultV1 = tryValidateAgainstSchema(rawResponse, reconstructionV1Schema);
const resultV1 = tryValidateAgainstSchema(
candidateResponse,
reconstructionV1Schema,
);
if (resultV1.valid) {
return buildSuccessResultV1(resultV1.data, OLLAMA_MODEL, duration, promptVersion);
return buildSuccessResultV1(
resultV1.data,
OLLAMA_MODEL,
duration,
promptVersion,
compatibility,
);
}
// ── Neither schema matched — partial failure ───────
@@ -93,17 +128,23 @@ export async function analyseScenario(scenario, opts = {}) {
resultV2.error ?? resultV1.error,
OLLAMA_MODEL,
duration,
promptVersion
promptVersion,
compatibility,
);
}
/** Attempt validation against a Zod schema */
function tryValidateAgainstSchema(data, schema) {
if (!schema.safeParse) {
return { valid: false, error: new Error("Schema does not support safeParse") };
return {
valid: false,
error: new Error("Schema does not support safeParse"),
};
}
const result = schema.safeParse(data);
return result.success ? { valid: true, data: result.data } : { valid: false, error: result.error };
return result.success
? { valid: true, data: result.data }
: { valid: false, error: result.error };
}
// ── Result builders ──────────────────────────────────
@@ -121,7 +162,15 @@ function buildErrorResponse(message, elapsed, statusCode = 500) {
};
}
function buildSuccessResultV2(data, model, duration, version) {
function buildCompatibilityDiagnostics(compatibility) {
return {
compatibilityApplied: compatibility.changesApplied.length > 0,
compatibilityChanges: compatibility.changesApplied,
compatibilityWarnings: compatibility.warnings,
};
}
function buildSuccessResultV2(data, model, duration, version, compatibility) {
return {
success: true,
validationStatus: "valid",
@@ -134,10 +183,11 @@ function buildSuccessResultV2(data, model, duration, version) {
evidence: data.evidence,
nextQuestion: data.nextQuestion,
errors: undefined,
...buildCompatibilityDiagnostics(compatibility),
};
}
function buildSuccessResultV1(data, model, duration, version) {
function buildSuccessResultV1(data, model, duration, version, compatibility) {
return {
success: true,
validationStatus: "valid",
@@ -150,14 +200,24 @@ function buildSuccessResultV1(data, model, duration, version) {
evidence: undefined,
nextQuestion: undefined,
errors: undefined,
...buildCompatibilityDiagnostics(compatibility),
};
}
function buildPartialResult(rawResp, error, model, duration, version) {
function buildPartialResult(
rawResp,
error,
model,
duration,
version,
compatibility,
) {
let errors = [];
if (error && typeof error.flatten === "function") {
errors = error.flatten().fieldErrors
? Object.entries(error.flatten().fieldErrors).flatMap(([k, v]) => [`${k}: ${v.join(", ")}`])
? Object.entries(error.flatten().fieldErrors).flatMap(([k, v]) => [
`${k}: ${v.join(", ")}`,
])
: [String(error)];
} else if (error) {
errors = [String(error).slice(0, 500)];
@@ -175,6 +235,7 @@ function buildPartialResult(rawResp, error, model, duration, version) {
evidence: undefined,
nextQuestion: undefined,
errors,
...buildCompatibilityDiagnostics(compatibility),
};
}
lib/graph/orchestrator.js
+3
View File
@@ -34,6 +34,9 @@ function buildDiagnostics({ analysis, graph, graphReferenceValidation }) {
nodeCount: graph?.nodes?.length ?? 0,
edgeCount: graph?.edges?.length ?? 0,
graphReferenceValidation,
compatibilityApplied: analysis?.compatibilityApplied ?? false,
compatibilityChanges: analysis?.compatibilityChanges ?? [],
compatibilityWarnings: analysis?.compatibilityWarnings ?? [],
};
}
lib/reconstruction/compatibility.js
+40
View File
@@ -0,0 +1,40 @@
function cloneJsonSafe(value) {
if (value == null) return value;
return JSON.parse(JSON.stringify(value));
}
export function normaliseAnalysisResponse(input) {
const normalised = cloneJsonSafe(input);
const changesApplied = [];
const warnings = [];
if (!normalised || typeof normalised !== "object") {
return { normalised: input, changesApplied, warnings };
}
if (Array.isArray(normalised.evidence)) {
normalised.evidence = normalised.evidence.map((record, index) => {
if (!record || typeof record !== "object") return record;
if (record.source === null) {
changesApplied.push({
path: ["evidence", index, "source"],
change: "Converted null source to undefined",
});
const { source: _removed, ...rest } = record;
return rest;
}
return record;
});
}
if (changesApplied.length > 0) {
warnings.push(
"Applied deterministic reconstruction compatibility normalisation",
);
}
return { normalised, changesApplied, warnings };
}
tests/graph/orchestrator.test.js
+26
View File
@@ -46,6 +46,9 @@ function makeAnalysisResult(overrides = {}) {
id: "q-1",
question: "What denominator is being used for the complaint rate?",
},
compatibilityApplied: false,
compatibilityChanges: [],
compatibilityWarnings: [],
...overrides,
};
}
@@ -171,6 +174,29 @@ describe("lib/graph/orchestrator startCase", () => {
expect(result.selectedQuestion).toBeNull();
});
it("includes compatibility diagnostics when provided by analysis", async () => {
mockAnalyseScenario.mockResolvedValue(
makeAnalysisResult({
compatibilityApplied: true,
compatibilityChanges: [
{
path: ["evidence", 0, "source"],
change: "Converted null source to undefined",
},
],
compatibilityWarnings: [
"Applied deterministic reconstruction compatibility normalisation",
],
}),
);
const { startCase } = await import("@/lib/graph/orchestrator.js");
const result = await startCase({ scenario: "Scenario text" });
expect(result.diagnostics.compatibilityApplied).toBe(true);
expect(result.diagnostics.compatibilityChanges).toHaveLength(1);
});
it("exports placeholder updateCase", async () => {
const { updateCase } = await import("@/lib/graph/orchestrator.js");
tests/reconstruction/compatibility.test.js
+179
View File
@@ -0,0 +1,179 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { normaliseAnalysisResponse } from "@/lib/reconstruction/compatibility.js";
const mockGenerateReconstruction = vi.fn();
vi.mock("@/lib/config.js", () => ({
getConfig: () => ({
ok: true,
config: {
OLLAMA_BASE_URL: "http://example.test",
OLLAMA_MODEL: "test-model",
},
}),
}));
vi.mock("@/lib/llm/provider.js", () => ({
getProvider: () => ({
generateReconstruction: (...args) => mockGenerateReconstruction(...args),
}),
}));
vi.mock("@/lib/reconstruction/prompt.js", () => ({
buildPrompt: async () => ({ prompt: "prompt", version: "v0.3" }),
PROMPT_VERSIONS: ["v0.1", "v0.2", "v0.3"],
DEFAULT_PROMPT_VERSION: "v0.3",
}));
describe("normaliseAnalysisResponse", () => {
beforeEach(() => {
vi.resetModules();
vi.clearAllMocks();
});
it("leaves already-valid responses unchanged", () => {
const input = {
evidence: [
{
id: "ev1",
description: "x",
evidenceType: "reported_statement",
confidence: "medium",
importance: "important",
source: "report",
},
],
};
const result = normaliseAnalysisResponse(input);
expect(result.normalised).toEqual(input);
expect(result.changesApplied).toEqual([]);
});
it("normalises null evidence source deterministically", () => {
const input = {
evidence: [
{
id: "ev1",
description: "x",
evidenceType: "reported_statement",
confidence: "medium",
importance: "important",
source: null,
},
],
};
const result = normaliseAnalysisResponse(input);
expect(result.normalised.evidence[0]).not.toHaveProperty("source");
expect(result.changesApplied).toHaveLength(1);
});
it("does not invent a next question", () => {
const input = { evidence: [] };
const result = normaliseAnalysisResponse(input);
expect(result.normalised.nextQuestion).toBeUndefined();
});
it("does not repair missing reasoning content", () => {
const input = { evidence: [{ source: null }] };
const result = normaliseAnalysisResponse(input);
expect(result.normalised.reconstruction).toBeUndefined();
});
});
describe("analyseScenario compatibility", () => {
it("succeeds when the only mismatch is null evidence source", async () => {
mockGenerateReconstruction.mockResolvedValue({
inputClassification: {
primaryType: "unexplained_change",
secondaryTypes: [],
reasoningModes: ["validate_measurement"],
classificationReason: "reason",
confidence: "medium",
},
reconstruction: {
summary: "summary",
actors: [],
systemsOrObjects: [],
expectedStates: [],
observedStates: [],
differences: [],
knownTransitions: [],
unexplainedTransitions: [],
contradictions: [],
importantUnknowns: [],
plausibleInterpretations: [],
},
evidence: [
{
id: "ev1",
description: "desc",
evidenceType: "reported_statement",
source: null,
attribution: null,
confidence: "medium",
importance: "important",
},
],
nextQuestion: {
id: "q1",
question: "What denominator?",
targets: ["observedStates"],
reason: "reason",
expectedInformationValue: "high",
reasoningMode: "validate_measurement",
},
});
const { analyseScenario } = await import("@/lib/analysis.js");
const result = await analyseScenario("Scenario text", {
promptVersion: "v0.3",
});
expect(result.success).toBe(true);
expect(result.compatibilityApplied).toBe(true);
expect(result.compatibilityChanges).toHaveLength(1);
expect(result.evidence[0]).not.toHaveProperty("source");
expect(result.nextQuestion.question).toBe("What denominator?");
});
it("still fails when required reasoning content is missing", async () => {
mockGenerateReconstruction.mockResolvedValue({
evidence: [
{
id: "ev1",
description: "desc",
evidenceType: "reported_statement",
source: null,
attribution: null,
confidence: "medium",
importance: "important",
},
],
});
const { analyseScenario } = await import("@/lib/analysis.js");
const result = await analyseScenario("Scenario text", {
promptVersion: "v0.3",
});
expect(result.success).toBe(false);
expect(result.compatibilityApplied).toBe(true);
expect(result.nextQuestion).toBeUndefined();
});
it("malformed JSON still fails", async () => {
mockGenerateReconstruction.mockResolvedValue("{not valid json");
const { analyseScenario } = await import("@/lib/analysis.js");
const result = await analyseScenario("Scenario text", {
promptVersion: "v0.3",
});
expect(result.success).toBe(false);
expect(result.compatibilityApplied).toBe(false);
});
});