test(confidence-engine): clarify controlled report experiment boundary
Retained deterministic input-control test proving that A/B apparatus correctly holds supportedEvidence, findings, plausibleInterpretations, scenarioState, and providerConfig constant while only varying unresolved question visibility. Added buildOverviewSynthesisPrompt identity proof that full vs filtered graph produce identical prompt text. Corrected handoff semantics: deterministic input filtering does NOT prove model output behaviour; live two-call semantic comparison remains required. Preserved prior imperfect live evidence with correct qualification about uncontrolled plausibleInterpretations divergence. Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -125,6 +125,39 @@ describe("Controlled A/B report epistemic comparison inputs", () => {
|
||||
expect(typeof a.unresolvedQuestions).toBe("object");
|
||||
});
|
||||
|
||||
// ── Prompt-builder identity proof (A/B produce identical SECTION A) ────
|
||||
|
||||
it("buildOverviewSynthesisPrompt produces identical prompt text when unknown nodes are present vs removed", async () => {
|
||||
const { buildOverviewSynthesisPrompt } = await import(
|
||||
"@/lib/graph/investigation-overview-synthesis.js"
|
||||
);
|
||||
const aInputs = buildABInputs(fixture, false);
|
||||
|
||||
const baseSg = fixture.situationGraph;
|
||||
const knownSupportedNodes = baseSg.nodes.filter(
|
||||
(n) => ["known", "supported"].includes(n.status)
|
||||
);
|
||||
|
||||
// Version A: filtered graph (unknowns removed)
|
||||
const promptA = buildOverviewSynthesisPrompt(
|
||||
{ ...baseSg, nodes: knownSupportedNodes },
|
||||
aInputs.supportedEvidence,
|
||||
aInputs.findings,
|
||||
aInputs.plausibleInterpretations
|
||||
);
|
||||
|
||||
// Version B: full graph (unknowns included) — should be identical because
|
||||
// buildOverviewSynthesisPrompt already filters to known+supported only
|
||||
const promptB = buildOverviewSynthesisPrompt(
|
||||
baseSg,
|
||||
aInputs.supportedEvidence,
|
||||
aInputs.findings,
|
||||
aInputs.plausibleInterpretations
|
||||
);
|
||||
|
||||
expect(promptA).toBe(promptB);
|
||||
});
|
||||
|
||||
it("fixed plausible interpretation carries NON-EVIDENCE epistemic status", () => {
|
||||
const a = buildABInputs(fixture, false);
|
||||
expect(a.plausibleInterpretations).toHaveLength(1);
|
||||
|
||||
Reference in New Issue
Block a user