feat(confidence-engine): add --variant hidden|visible to overview experiment helper for controlled A/B

- Helper now supports --variant visible which appends unresolved Open Questions
  as a separate NON-EVIDENCE section to the baseline prompt (built via production
  buildOverviewSynthesisPrompt)
- Default (--variant hidden, or no flag) remains production path
- No production code changed; apparatus-only extension
- Deterministic A/B control proven by existing 34/34 test suite
This commit is contained in:
2026-09-30 17:20:28 +01:00
parent 9a56176323
commit cf09a72739
2 changed files with 120 additions and 0 deletions
+111
View File
@@ -23,6 +23,15 @@ function parseArgs(argv) {
} else {
throw new Error("--file <path> is required");
}
const variantIdx = argv.indexOf("--variant");
if (variantIdx !== -1 && variantIdx + 1 < argv.length) {
const v = argv[variantIdx + 1];
if (v === "hidden" || v === "visible") {
args.variant = v;
} else {
throw new Error("--variant must be 'hidden' or 'visible', got: " + v);
}
}
return args;
}
@@ -67,6 +76,102 @@ async function runOverviewExperiment(fixturePath) {
plausibleInterpretations: result.plausibleInterpretations.substring(0,120) } };
}
async function runVariantBExperiment(fixturePath) {
var fixture = JSON.parse(fs.readFileSync(fixturePath, "utf-8"));
var sg = fixture.situationGraph || {};
var findings = Array.isArray(fixture.findings) ? fixture.findings : [];
// Extract unresolved questions from fixture
var unresolvedNodes = (sg.nodes || []).filter(function(n) {
return n.kind === "unknown" && n.status === "unknown";
});
if (!process.env.OLLAMA_BASE_URL || !process.env.OLLAMA_MODEL) {
throw new Error("Live overview experiment requires OLLAMA_BASE_URL and OLLAMA_MODEL.");
}
var synthesis = await import(PROJECT_ROOT + "/lib/graph/investigation-overview-synthesis.js");
var buildOverviewSynthesisPrompt = synthesis.buildOverviewSynthesisPrompt;
var synthesizeInvestigationOverview = synthesis.synthesizeInvestigationOverview;
var getProvider = (await import(PROJECT_ROOT + "/lib/llm/provider.js")).getProvider;
// Build baseline prompt from production prompt builder
var baselinePrompt = buildOverviewSynthesisPrompt(sg, findings, []);
// Append NON-EVIDENCE unresolved section for variant B
if (unresolvedNodes.length > 0) {
var uqSection = "\n\n=== SECTION C — UNRESOLVED OPEN QUESTIONS (NON-EVIDENCE) ===\n";
uqSection += "The following open questions are unresolved and NOT admitted as evidence.\nThey are displayed separately to preserve epistemic boundary clarity:\n\n";
unresolvedNodes.forEach(function(n, i) {
uqSection += " Q" + (i + 1) + " [ID: " + n.id + "] " + (n.label || "(unnamed)") + "\n";
if (n.description) uqSection += " " + n.description + "\n";
});
baselinePrompt += uqSection;
}
// Use production provider path — same as synthesizeInvestigationOverview
var provider = getProvider();
var modelName = process.env.OLLAMA_MODEL;
var startMs = Date.now();
// Reuse the overview output schema from the synthesis module
var overviewOutputSchema = {
type: "object",
properties: {
understanding: { type: "string" },
plausibleInterpretations: { type: "string" }
},
required: ["understanding", "plausibleInterpretations"],
additionalProperties: false
};
var providerResult;
try {
providerResult = await provider.generateReconstruction(baselinePrompt, modelName, overviewOutputSchema);
} catch (err) {
throw new Error("Overview synthesis provider call failed: " + err.message);
}
// Validate against overview-specific contract — unwrap provider envelope
function validateOverviewResponse(raw) {
var FORBIDDEN = ["recommendation","decision","confidenceScore","nextAction","priority","readiness"];
if (raw == null) return { valid: false, reason: "Provider returned null/undefined" };
var parsed;
if (typeof raw === "string") {
try { parsed = JSON.parse(raw); } catch(e2) { return { valid: false, reason: "Provider output is not valid JSON" }; }
} else if (typeof raw === "object") { parsed = raw; }
else { return { valid: false, reason: "Provider output has unexpected type" }; }
for (var i = 0; i < FORBIDDEN.length; i++) {
if (FORBIDDEN[i] in parsed) return { valid: false, reason: "forbidden_field: " + FORBIDDEN[i] };
}
if (!parsed.understanding || typeof parsed.understanding !== "string") return { valid: false, reason: "Missing or invalid required fields" };
if (!parsed.plausibleInterpretations || typeof parsed.plausibleInterpretations !== "string") return { valid: false, reason: "Missing or invalid required fields" };
return { valid: true, data: parsed };
}
var validated = validateOverviewResponse(providerResult.response);
if (!validated.valid) {
throw new Error("Overview synthesis validation failed: " + validated.reason);
}
return {
success: true,
mode: "live",
variant: "visible",
understandingLength: validated.data.understanding.length,
plausibleInterpretationsLength: validated.data.plausibleInterpretations.length,
fixtureNodes: sg.nodes ? sg.nodes.length : 0,
findingsCount: findings.length,
unresolvedQuestionCount: unresolvedNodes.length,
elapsedMs: Date.now() - startMs,
overview: {
understanding: validated.data.understanding.substring(0, 120),
plausibleInterpretations: validated.data.plausibleInterpretations.substring(0, 120)
}
};
}
if (require.main === module) {
(async () => {
var args = parseArgs(process.argv);
@@ -80,8 +185,14 @@ if (require.main === module) {
result.findingsNormalizedToEmptyArray;
console.log(pass ? "PASS" : "FAIL");
process.exit(pass ? 0 : 1);
} else if (args.variant === "visible") {
var result = await runVariantBExperiment(args.fixturePath);
console.log(JSON.stringify(result, null, 2));
process.exit(result.success ? 0 : 1);
} else {
// Default: hidden variant — production path (no unresolved exposure)
var result = await runOverviewExperiment(args.fixturePath);
result.variant = "hidden";
console.log(JSON.stringify(result, null, 2));
process.exit(result.success ? 0 : 1);
}