test(confidence-engine): cover CU diagnostics and investigation checkpoints

This commit is contained in:
2026-10-08 06:45:37 +01:00
parent 20b416dcb6
commit ddd1c7bd6f
2 changed files with 106 additions and 1 deletions
@@ -635,4 +635,109 @@ describe("synthesizeCurrentUnderstanding — full seam", () => {
)
).rejects.toThrow(/Synthesis validation failed/);
});
// ── Batch 3 Iteration 1: CU synthesis observability (sanitized diagnostic) ──
it("validation failure exposes sanitizedDiagnostic with safe shape fields — no content leaked", async () => {
const fake = makeFakeProvider(async () => JSON.stringify({ wrongField: "value" }));
try {
await synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
);
} catch (err) {
expect(err.message).toMatch(/Synthesis validation failed/);
expect(err.statusCode).toBe(502);
expect(err.sanitizedDiagnostic).toBeDefined();
expect(err.sanitizedDiagnostic).toBeInstanceOf(Object);
const d = err.sanitizedDiagnostic;
// T1: providerApiPath exposed as string or null (safe)
expect(d.providerApiPath).toBeDefined();
expect(typeof d.providerApiPath).toMatch(/string|number|boolean|undefined|null/);
// T2: responseType is a safe primitive exposing typeof the response
expect(typeof d.responseType).toBe("string");
// T3: responseIsNull boolean (safe)
expect(typeof d.responseIsNull).toBe("boolean");
// T4: hasCurrentUnderstanding boolean — proves structural detection works
expect(typeof d.hasCurrentUnderstanding).toBe("boolean");
// T5: currentUnderstandingType is null when field absent (safe)
expect(d.currentUnderstandingType).toBeNull();
// T6: currentUnderstandingLength is null when field absent (safe)
expect(d.currentUnderstandingLength).toBeNull();
// T7: responseKeys reveals shape but never CU text — wrongField is safe key
expect(Array.isArray(d.responseKeys)).toBe(true);
expect(d.responseKeys).toContain("wrongField");
}
});
it("sanitizedDiagnostic does NOT include actual CU narrative content", async () => {
const fake = makeFakeProvider(async () => JSON.stringify({ currentUnderstanding: "Should not appear in diagnostic" }));
try {
await synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
);
} catch (err) {
expect(err.sanitizedDiagnostic).toBeDefined();
const d = err.sanitizedDiagnostic;
// The diagnostic should expose length but not content
expect(typeof d.currentUnderstandingLength).toBe("number");
// Length 43 for "Should not appear in diagnostic" — if it were wrongField the field wouldn't exist so hasCU would be false and length would be null
expect(d.hasCurrentUnderstanding).toBe(true);
// Crucially: actual content string must NOT appear anywhere in diagnostic
const diagStr = JSON.stringify(d);
expect(diagStr).not.toContain("Should not appear");
}
});
it("provider returns response object (already parsed) with valid currentUnderstanding — no synthetic test", async () => {
const fake = makeFakeProvider(); // default: returns JSON stringified valid output
const result = await synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings: [workingFinding()] },
{ provider: fake }
);
expect(result).toEqual({ currentUnderstanding: "Synthesized output" });
});
it("validateSynthesisResponse with response that is a plain object (not string) — still valid", async () => {
const fake = makeFakeProvider(async () => ({ currentUnderstanding: "from-object" }));
const result = await synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
);
expect(result).toEqual({ currentUnderstanding: "from-object" });
});
it("responseKeys exposed in diagnostic includes top-level keys of any response object", async () => {
const fake = makeFakeProvider(async () => JSON.stringify({ foo: 1, bar: "baz", currentUnderstanding: "x" }));
try {
await synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
);
} catch (err) {
expect(err.sanitizedDiagnostic.responseKeys).toEqual(["bar", "currentUnderstanding", "foo"]);
}
});
it("validation success returns normal result — no diagnostic attached", async () => {
const fake = makeFakeProvider();
try {
await synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
);
} catch (err) {
// Should not reach here — success path should not throw
expect(err).toBeUndefined();
}
});
});
@@ -685,7 +685,7 @@ async function runSyntheticTests() {
}
// Run synthetic tests and exit with combined result
var synPromise = runSyntheticTests().then(function(synthFail) {
runSyntheticTests().then(function(synthFail) {
var totalFailures = failures + synthFail;
process.exit(totalFailures > 0 ? 1 : 0);
});