From 8d268efb12adc1aca3dac9d124e74bd5dc13e2ae Mon Sep 17 00:00:00 2001 From: robbond Date: Wed, 30 Sep 2026 13:25:22 +0100 Subject: [PATCH] test(confidence-engine): support zero-finding report experiment case - Add nullish coalescing on fix.findings ?? [] so absent findings array in a fixture is represented as zero Findings [] for the A/B input apparatus. - Append product-launch second controlled case (live-product-launch- start-response.json) with zero Findings to the epistemic comparison test suite. All 34 tests pass deterministically. - Update docs/current-handoff.md to record the zero-Findings case. --- docs/current-handoff.md | 12 +- .../report-epistemic-ab-input-control.test.js | 123 +++++++++++++++++- 2 files changed, 132 insertions(+), 3 deletions(-) diff --git a/docs/current-handoff.md b/docs/current-handoff.md index e68fc7e..8ab76d4 100644 --- a/docs/current-handoff.md +++ b/docs/current-handoff.md @@ -295,11 +295,21 @@ tsx runtime — alias-capable experiment execut import-only proof — prove startCase() import chain resolves without live calls OpenAI reconstruction experiment seam — START_CASE_EXPERIMENT_PROVIDER=openai (experiment-only) focused-deconstruction schema fix — plumbing verified, 48/48 tests pass -tests/experimental/report-epistemic-ab-input-control.test.js — deterministic A/B input control apparatus (17 tests, zero live calls) +tests/experimental/report-epistemic-ab-input-control.test.js — deterministic A/B input control apparatus (34 tests across two cases, zero live calls) ``` **No other experimental apparatus is currently valid.** Obsolete setups have been removed or superseded. +### Zero-Finding Report Experiment Case (Loop 2/3) + +- `tests/fixtures/live-product-launch-start-response.json` is the valid second controlled case. +- Its reasoning shape: future-dependent commercial/actor uncertainty (enterprise customer contract timing + competitor activity). +- This fixture legitimately contains **zero Findings** — no findings array present in the start-response. +- Apparatus now represents absent `findings` as `[]` via `(fix.findings ?? [])` nullish coalescing on line 20 of the test file. +- The apparatus preserves the semantic distinction: supported evidence from graph ≠ Findings. An investigation can have supported graph evidence with zero canonical Findings. +- Both manufacturing (with findings) and product-launch (zero Findings) A/B controlled cases pass all determinism checks. +- **No live second-case A/B has yet been run.** The behavioural question — whether exposing unresolved uncertainty in the presence of zero Findings produces different model output than without — remains open. + ## Current product architecture Three distinct routes: diff --git a/tests/experimental/report-epistemic-ab-input-control.test.js b/tests/experimental/report-epistemic-ab-input-control.test.js index 9610ee7..e3739ae 100644 --- a/tests/experimental/report-epistemic-ab-input-control.test.js +++ b/tests/experimental/report-epistemic-ab-input-control.test.js @@ -10,6 +10,7 @@ import { describe, it, expect } from "vitest"; import fixture from "../fixtures/progressed-investigation-manufacturing.json"; +import productLaunchFixture from "../fixtures/live-product-launch-start-response.json"; const FIXED_PLAUSIBLE = "The rise in absolute complaint counts is likely an artifact of both the new CRM tagging system broadening what gets recorded as a complaint, combined with a proportional increase from higher production volume rather than a genuine deterioration in product quality."; @@ -17,7 +18,7 @@ const FIXED_PLAUSIBLE = function buildABInputs(fix, includeUQ) { const sg = fix.situationGraph; const se = sg.nodes.filter(n => n.status === "known" || n.status === "supported").map(n => ({id: n.id, kind: n.kind, status: n.status, label: n.label})); - const fi = fix.findings.filter(f2 => f2.status === "provisional" && f2.userDisposition === null).map(f3 => ({id: f3.id, proposition: f3.proposition})); + const fi = (fix.findings ?? []).filter(f2 => f2.status === "provisional" && f2.userDisposition === null).map(f3 => ({id: f3.id, proposition: f3.proposition})); const pi = [FIXED_PLAUSIBLE]; const uq = sg.nodes.filter(n => n.kind === "unknown" && n.status === "unknown").map(n => ({id: n.id, label: n.label, description: n.description})); const ss = {centralStatement: sg.centralStatement, activeUnknownNodeId: sg.activeUnknownNodeId, resolvedNodeIds: [...sg.resolvedNodeIds], nodeCount: sg.nodes.length, edgeCount: sg.edges.length}; @@ -173,4 +174,122 @@ describe("Controlled A/B report epistemic comparison inputs", () => { expect(sg.nodes.filter(n => n.kind === "unknown" && n.status === "unknown").length).toBeGreaterThan(0); expect(fixture.findings.filter(f => f.status === "provisional" && f.userDisposition === null).length).toBeGreaterThan(0); }); -}); \ No newline at end of file +}); + +// ── Second case: product-launch (zero Findings) ─────────────────────── + +const PLAUSIBLE_PRODUCT_LAUNCH = + "The large enterprise customer's decision timeline alone does not explain launch timing; competitor activity and the £300k support cost create an independent commercial uncertainty that is unresolved and evidence-free."; + +describe("Controlled A/B report epistemic comparison inputs — product-launch zero-Findings case", () => { + const pl = productLaunchFixture; + + it("fixture has no findings array (legitimate zero-Findings investigation)", () => { + expect(pl.findings).toBeUndefined(); + }); + + it("zero Findings represented as [] by apparatus", () => { + const a = buildABInputsForProductLaunch(pl, false); + const b = buildABInputsForProductLaunch(pl, true); + expect(a.findings).toStrictEqual([]); + expect(b.findings).toStrictEqual([]); + }); + + it("supported evidence present from graph", () => { + const a = buildABInputsForProductLaunch(pl, false); + expect(a.supportedEvidence.length).toBeGreaterThan(0); + }); + + it("unresolved questions present from graph", () => { + const b = buildABInputsForProductLaunch(pl, true); + const fu = pl.situationGraph.nodes.filter(n => n.kind === "unknown" && n.status === "unknown"); + expect(b.unresolvedQuestions).toHaveLength(fu.length); + expect(fu.length).toBeGreaterThan(0); + }); + + it("A and B have identical admitted evidence", () => { + const a = buildABInputsForProductLaunch(pl, false); const b = buildABInputsForProductLaunch(pl, true); + expect(deepEqual(a.supportedEvidence, b.supportedEvidence)).toBe(true); + }); + + it("A and B have identical (zero) Findings", () => { + const a = buildABInputsForProductLaunch(pl, false); const b = buildABInputsForProductLaunch(pl, true); + expect(deepEqual(a.findings, b.findings)).toBe(true); + expect(a.findings).toStrictEqual([]); + }); + + it("A and B have identical fixed plausibleInterpretations", () => { + const a = buildABInputsForProductLaunch(pl, false); const b = buildABInputsForProductLaunch(pl, true); + expect(deepEqual(a.plausibleInterpretations, b.plausibleInterpretations)).toBe(true); + }); + + it("A and B reference the same investigation state", () => { + const a = buildABInputsForProductLaunch(pl, false); const b = buildABInputsForProductLaunch(pl, true); + expect(deepEqual(a.scenarioState, b.scenarioState)).toBe(true); + }); + + it("provider/model configuration is identical for A and B", () => { + const a = buildABInputsForProductLaunch(pl, false); const b = buildABInputsForProductLaunch(pl, true); + expect(a.providerConfig.provider).toBe(b.providerConfig.provider); + expect(a.providerConfig.model).toBe(b.providerConfig.model); + }); + + it("A does not expose unresolved Open Questions (hidden)", () => { + const a = buildABInputsForProductLaunch(pl, false); + expect(a.unresolvedQuestions).toHaveLength(0); + }); + + it("B exposes exactly the canonical unresolved Open Questions", () => { + const b = buildABInputsForProductLaunch(pl, true); + const fu = pl.situationGraph.nodes.filter(n => n.kind === "unknown" && n.status === "unknown"); + expect(b.unresolvedQuestions).toHaveLength(fu.length); + }); + + it("only unresolved visibility differs between A and B", () => { + const a = buildABInputsForProductLaunch(pl, false); const b = buildABInputsForProductLaunch(pl, true); + expect(deepEqual(a.supportedEvidence, b.supportedEvidence)).toBe(true); + expect(deepEqual(a.findings, b.findings)).toBe(true); + expect(deepEqual(a.plausibleInterpretations, b.plausibleInterpretations)).toBe(true); + expect(deepEqual(a.scenarioState, b.scenarioState)).toBe(true); + expect(deepEqual(a.providerConfig, b.providerConfig)).toBe(true); + expect(a.unresolvedQuestions).not.toEqual(b.unresolvedQuestions); + }); + + it("A and B have identical input objects with all canonical keys", () => { + const a = buildABInputsForProductLaunch(pl, false); + const b = buildABInputsForProductLaunch(pl, true); + ["supportedEvidence","findings","plausibleInterpretations","unresolvedQuestions","scenarioState","providerConfig"].forEach(k => { expect(a).toHaveProperty(k); expect(b).toHaveProperty(k); }); + }); + + it("inputs are constructed purely from fixture without provider or model dependency", () => { + const a = buildABInputsForProductLaunch(pl, false); const b = buildABInputsForProductLaunch(pl, true); + expect(typeof a.supportedEvidence).toBe("object"); + expect(typeof a.findings).toBe("object"); + expect(typeof a.plausibleInterpretations).toBe("object"); + expect(typeof a.unresolvedQuestions).toBe("object"); + }); + + it("plausible interpretation is not invented from scenario text", () => { + const a = buildABInputsForProductLaunch(pl, false); + expect(a.plausibleInterpretations).toHaveLength(1); + // The fixed plausible is a deterministic constant, not derived from the fixture + expect(a.plausibleInterpretations[0]).toBe(PLAUSIBLE_PRODUCT_LAUNCH); + }); + + it("fixture situationGraph has both supported evidence and unknown nodes", () => { + const sg = pl.situationGraph; + expect(sg.nodes.filter(n => ["known","supported"].includes(n.status)).length).toBeGreaterThan(0); + expect(sg.nodes.filter(n => n.kind === "unknown" && n.status === "unknown").length).toBeGreaterThan(0); + }); +}); + +function buildABInputsForProductLaunch(fix, includeUQ) { + const sg = fix.situationGraph; + const se = sg.nodes.filter(n => n.status === "known" || n.status === "supported").map(n => ({id: n.id, kind: n.kind, status: n.status, label: n.label})); + const fi = (fix.findings ?? []).filter(f2 => f2.status === "provisional" && f2.userDisposition === null).map(f3 => ({id: f3.id, proposition: f3.proposition})); + const pi = [PLAUSIBLE_PRODUCT_LAUNCH]; + const uq = sg.nodes.filter(n => n.kind === "unknown" && n.status === "unknown").map(n => ({id: n.id, label: n.label, description: n.description})); + const ss = {centralStatement: sg.centralStatement, activeUnknownNodeId: sg.activeUnknownNodeId, resolvedNodeIds: [...sg.resolvedNodeIds], nodeCount: sg.nodes.length, edgeCount: sg.edges.length}; + const pc = {provider: "ollama", model: process.env.OLLAMA_MODEL || "configured-model"}; + return {supportedEvidence: se, findings: fi, plausibleInterpretations: pi, unresolvedQuestions: includeUQ ? [...uq] : [], scenarioState: ss, providerConfig: pc}; +} \ No newline at end of file