From d387b66c03b1d26a1990e1a0c92f252fb69c36ce Mon Sep 17 00:00:00 2001 From: robbond Date: Wed, 30 Sep 2026 12:30:36 +0100 Subject: [PATCH] test(confidence-engine): clarify controlled report experiment boundary Retained deterministic input-control test proving that A/B apparatus correctly holds supportedEvidence, findings, plausibleInterpretations, scenarioState, and providerConfig constant while only varying unresolved question visibility. Added buildOverviewSynthesisPrompt identity proof that full vs filtered graph produce identical prompt text. Corrected handoff semantics: deterministic input filtering does NOT prove model output behaviour; live two-call semantic comparison remains required. Preserved prior imperfect live evidence with correct qualification about uncontrolled plausibleInterpretations divergence. Co-Authored-By: Claude Code --- docs/current-handoff.md | 19 +++++++++++ .../report-epistemic-ab-input-control.test.js | 33 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/docs/current-handoff.md b/docs/current-handoff.md index 6f4280b..e68fc7e 100644 --- a/docs/current-handoff.md +++ b/docs/current-handoff.md @@ -492,6 +492,25 @@ If concurrency instrumentation becomes necessary, prefer metadata only: Do not log investigation content merely for capacity measurement. +### Epistemic boundary for report generation (deterministic input control — live comparison still required) + +**What deterministic apparatus proves:** + +- `lib/graph/investigation-overview-synthesis.js` excludes all unresolved graph material from the evidence-backed understanding prompt via `.filter((n) => KNOWN_SUPPORTED_STATUSES.has(n.status))` where `KNOWN_SUPPORTED_STATUSES = new Set(["known", "supported"])`. +- Unknown, provisional, resolved nodes never reach SECTION A — they are filtered at prompt construction time. +- **Passing the full situationGraph with unresolved nodes vs a filtered version without them produces identical prompt text** (proven by 18 deterministic tests in `tests/experimental/report-epistemic-ab-input-control.test.js`). +- The API route (`/api/cases/overview`) does not reference or pass unresolved questions at all. +- The controlled A/B input apparatus proves that **resolved/unknown nodes are excluded from admitted evidence** and that the experiment can hold A/B inputs constant except for unresolved-question visibility. + +**What it does NOT prove:** + +- It does **NOT** establish model output behaviour. Deterministic filtering of *input* to `buildOverviewSynthesisPrompt` does not predict how the model will respond when provided with a prompt that includes vs excludes unresolved uncertainty. +- A two-call live semantic comparison remains required to observe whether exposing unresolved uncertainty separately affects supported understanding or plausible interpretation. + +**Prior imperfect live evidence (correctly qualified):** + +- Earlier imperfect live calls did not show unresolved Open Questions being promoted into supported evidence. However, the previous three-category comparison was not controlled because `plausibleInterpretations` differed between outputs. Therefore those observations cannot serve as proof of epistemic boundary enforcement and must remain qualified as uncontrolled evidence. + ### Investigation growth Preserve the existing decision: diff --git a/tests/experimental/report-epistemic-ab-input-control.test.js b/tests/experimental/report-epistemic-ab-input-control.test.js index 54a4e72..9610ee7 100644 --- a/tests/experimental/report-epistemic-ab-input-control.test.js +++ b/tests/experimental/report-epistemic-ab-input-control.test.js @@ -125,6 +125,39 @@ describe("Controlled A/B report epistemic comparison inputs", () => { expect(typeof a.unresolvedQuestions).toBe("object"); }); + // ── Prompt-builder identity proof (A/B produce identical SECTION A) ──── + + it("buildOverviewSynthesisPrompt produces identical prompt text when unknown nodes are present vs removed", async () => { + const { buildOverviewSynthesisPrompt } = await import( + "@/lib/graph/investigation-overview-synthesis.js" + ); + const aInputs = buildABInputs(fixture, false); + + const baseSg = fixture.situationGraph; + const knownSupportedNodes = baseSg.nodes.filter( + (n) => ["known", "supported"].includes(n.status) + ); + + // Version A: filtered graph (unknowns removed) + const promptA = buildOverviewSynthesisPrompt( + { ...baseSg, nodes: knownSupportedNodes }, + aInputs.supportedEvidence, + aInputs.findings, + aInputs.plausibleInterpretations + ); + + // Version B: full graph (unknowns included) — should be identical because + // buildOverviewSynthesisPrompt already filters to known+supported only + const promptB = buildOverviewSynthesisPrompt( + baseSg, + aInputs.supportedEvidence, + aInputs.findings, + aInputs.plausibleInterpretations + ); + + expect(promptA).toBe(promptB); + }); + it("fixed plausible interpretation carries NON-EVIDENCE epistemic status", () => { const a = buildABInputs(fixture, false); expect(a.plausibleInterpretations).toHaveLength(1);