From cf09a72739fef505716ca0b00c335643230970cc Mon Sep 17 00:00:00 2001 From: robbond Date: Wed, 30 Sep 2026 17:20:28 +0100 Subject: [PATCH] feat(confidence-engine): add --variant hidden|visible to overview experiment helper for controlled A/B - Helper now supports --variant visible which appends unresolved Open Questions as a separate NON-EVIDENCE section to the baseline prompt (built via production buildOverviewSynthesisPrompt) - Default (--variant hidden, or no flag) remains production path - No production code changed; apparatus-only extension - Deterministic A/B control proven by existing 34/34 test suite --- docs/current-handoff.md | 9 ++ scripts/overview-experiment-helper.cjs | 111 +++++++++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/docs/current-handoff.md b/docs/current-handoff.md index 17d3411..d5ffec9 100644 --- a/docs/current-handoff.md +++ b/docs/current-handoff.md @@ -302,6 +302,15 @@ scripts/overview-experiment-helper.cjs — canonical overview live helper **No other experimental apparatus is currently valid.** Obsolete setups have been removed or superseded. +### Epistemic A/B overview apparatus extension (Loop 4/1) + +- `scripts/overview-experiment-helper.cjs` extended with `--variant hidden|visible`: + - **hidden** (default): production path — unresolved questions filtered by prompt builder + - **visible**: appends unresolved questions as separate NON-EVIDENCE section to baseline prompt +- Apparatus invariant proven deterministically: A/B evidence, findings, plausibleInterpretations, scenario state are identical; only unresolved visibility differs. +- Production `lib/graph/investigation-overview-synthesis.js` unchanged. +- **No live A/B calls executed yet.** Next boundary: Call A (hidden) with product-launch fixture. + ### Zero-Finding Report Experiment Case (Loop 2/3) - `tests/fixtures/live-product-launch-start-response.json` is the valid second controlled case. diff --git a/scripts/overview-experiment-helper.cjs b/scripts/overview-experiment-helper.cjs index 07e8975..d344e63 100644 --- a/scripts/overview-experiment-helper.cjs +++ b/scripts/overview-experiment-helper.cjs @@ -23,6 +23,15 @@ function parseArgs(argv) { } else { throw new Error("--file is required"); } + const variantIdx = argv.indexOf("--variant"); + if (variantIdx !== -1 && variantIdx + 1 < argv.length) { + const v = argv[variantIdx + 1]; + if (v === "hidden" || v === "visible") { + args.variant = v; + } else { + throw new Error("--variant must be 'hidden' or 'visible', got: " + v); + } + } return args; } @@ -67,6 +76,102 @@ async function runOverviewExperiment(fixturePath) { plausibleInterpretations: result.plausibleInterpretations.substring(0,120) } }; } +async function runVariantBExperiment(fixturePath) { + var fixture = JSON.parse(fs.readFileSync(fixturePath, "utf-8")); + var sg = fixture.situationGraph || {}; + var findings = Array.isArray(fixture.findings) ? fixture.findings : []; + + // Extract unresolved questions from fixture + var unresolvedNodes = (sg.nodes || []).filter(function(n) { + return n.kind === "unknown" && n.status === "unknown"; + }); + + if (!process.env.OLLAMA_BASE_URL || !process.env.OLLAMA_MODEL) { + throw new Error("Live overview experiment requires OLLAMA_BASE_URL and OLLAMA_MODEL."); + } + + var synthesis = await import(PROJECT_ROOT + "/lib/graph/investigation-overview-synthesis.js"); + var buildOverviewSynthesisPrompt = synthesis.buildOverviewSynthesisPrompt; + var synthesizeInvestigationOverview = synthesis.synthesizeInvestigationOverview; + var getProvider = (await import(PROJECT_ROOT + "/lib/llm/provider.js")).getProvider; + + // Build baseline prompt from production prompt builder + var baselinePrompt = buildOverviewSynthesisPrompt(sg, findings, []); + + // Append NON-EVIDENCE unresolved section for variant B + if (unresolvedNodes.length > 0) { + var uqSection = "\n\n=== SECTION C — UNRESOLVED OPEN QUESTIONS (NON-EVIDENCE) ===\n"; + uqSection += "The following open questions are unresolved and NOT admitted as evidence.\nThey are displayed separately to preserve epistemic boundary clarity:\n\n"; + unresolvedNodes.forEach(function(n, i) { + uqSection += " Q" + (i + 1) + " [ID: " + n.id + "] " + (n.label || "(unnamed)") + "\n"; + if (n.description) uqSection += " " + n.description + "\n"; + }); + baselinePrompt += uqSection; + } + + // Use production provider path — same as synthesizeInvestigationOverview + var provider = getProvider(); + var modelName = process.env.OLLAMA_MODEL; + var startMs = Date.now(); + + // Reuse the overview output schema from the synthesis module + var overviewOutputSchema = { + type: "object", + properties: { + understanding: { type: "string" }, + plausibleInterpretations: { type: "string" } + }, + required: ["understanding", "plausibleInterpretations"], + additionalProperties: false + }; + + var providerResult; + try { + providerResult = await provider.generateReconstruction(baselinePrompt, modelName, overviewOutputSchema); + } catch (err) { + throw new Error("Overview synthesis provider call failed: " + err.message); + } + + // Validate against overview-specific contract — unwrap provider envelope + function validateOverviewResponse(raw) { + var FORBIDDEN = ["recommendation","decision","confidenceScore","nextAction","priority","readiness"]; + if (raw == null) return { valid: false, reason: "Provider returned null/undefined" }; + var parsed; + if (typeof raw === "string") { + try { parsed = JSON.parse(raw); } catch(e2) { return { valid: false, reason: "Provider output is not valid JSON" }; } + } else if (typeof raw === "object") { parsed = raw; } + else { return { valid: false, reason: "Provider output has unexpected type" }; } + + for (var i = 0; i < FORBIDDEN.length; i++) { + if (FORBIDDEN[i] in parsed) return { valid: false, reason: "forbidden_field: " + FORBIDDEN[i] }; + } + if (!parsed.understanding || typeof parsed.understanding !== "string") return { valid: false, reason: "Missing or invalid required fields" }; + if (!parsed.plausibleInterpretations || typeof parsed.plausibleInterpretations !== "string") return { valid: false, reason: "Missing or invalid required fields" }; + return { valid: true, data: parsed }; + } + + var validated = validateOverviewResponse(providerResult.response); + if (!validated.valid) { + throw new Error("Overview synthesis validation failed: " + validated.reason); + } + + return { + success: true, + mode: "live", + variant: "visible", + understandingLength: validated.data.understanding.length, + plausibleInterpretationsLength: validated.data.plausibleInterpretations.length, + fixtureNodes: sg.nodes ? sg.nodes.length : 0, + findingsCount: findings.length, + unresolvedQuestionCount: unresolvedNodes.length, + elapsedMs: Date.now() - startMs, + overview: { + understanding: validated.data.understanding.substring(0, 120), + plausibleInterpretations: validated.data.plausibleInterpretations.substring(0, 120) + } + }; +} + if (require.main === module) { (async () => { var args = parseArgs(process.argv); @@ -80,8 +185,14 @@ if (require.main === module) { result.findingsNormalizedToEmptyArray; console.log(pass ? "PASS" : "FAIL"); process.exit(pass ? 0 : 1); + } else if (args.variant === "visible") { + var result = await runVariantBExperiment(args.fixturePath); + console.log(JSON.stringify(result, null, 2)); + process.exit(result.success ? 0 : 1); } else { + // Default: hidden variant — production path (no unresolved exposure) var result = await runOverviewExperiment(args.fixturePath); + result.variant = "hidden"; console.log(JSON.stringify(result, null, 2)); process.exit(result.success ? 0 : 1); }