feat(confidence-engine): add --variant hidden|visible to overview experiment helper for controlled A/B
- Helper now supports --variant visible which appends unresolved Open Questions as a separate NON-EVIDENCE section to the baseline prompt (built via production buildOverviewSynthesisPrompt) - Default (--variant hidden, or no flag) remains production path - No production code changed; apparatus-only extension - Deterministic A/B control proven by existing 34/34 test suite
This commit is contained in:
@@ -23,6 +23,15 @@ function parseArgs(argv) {
|
||||
} else {
|
||||
throw new Error("--file <path> is required");
|
||||
}
|
||||
const variantIdx = argv.indexOf("--variant");
|
||||
if (variantIdx !== -1 && variantIdx + 1 < argv.length) {
|
||||
const v = argv[variantIdx + 1];
|
||||
if (v === "hidden" || v === "visible") {
|
||||
args.variant = v;
|
||||
} else {
|
||||
throw new Error("--variant must be 'hidden' or 'visible', got: " + v);
|
||||
}
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
@@ -67,6 +76,102 @@ async function runOverviewExperiment(fixturePath) {
|
||||
plausibleInterpretations: result.plausibleInterpretations.substring(0,120) } };
|
||||
}
|
||||
|
||||
async function runVariantBExperiment(fixturePath) {
|
||||
var fixture = JSON.parse(fs.readFileSync(fixturePath, "utf-8"));
|
||||
var sg = fixture.situationGraph || {};
|
||||
var findings = Array.isArray(fixture.findings) ? fixture.findings : [];
|
||||
|
||||
// Extract unresolved questions from fixture
|
||||
var unresolvedNodes = (sg.nodes || []).filter(function(n) {
|
||||
return n.kind === "unknown" && n.status === "unknown";
|
||||
});
|
||||
|
||||
if (!process.env.OLLAMA_BASE_URL || !process.env.OLLAMA_MODEL) {
|
||||
throw new Error("Live overview experiment requires OLLAMA_BASE_URL and OLLAMA_MODEL.");
|
||||
}
|
||||
|
||||
var synthesis = await import(PROJECT_ROOT + "/lib/graph/investigation-overview-synthesis.js");
|
||||
var buildOverviewSynthesisPrompt = synthesis.buildOverviewSynthesisPrompt;
|
||||
var synthesizeInvestigationOverview = synthesis.synthesizeInvestigationOverview;
|
||||
var getProvider = (await import(PROJECT_ROOT + "/lib/llm/provider.js")).getProvider;
|
||||
|
||||
// Build baseline prompt from production prompt builder
|
||||
var baselinePrompt = buildOverviewSynthesisPrompt(sg, findings, []);
|
||||
|
||||
// Append NON-EVIDENCE unresolved section for variant B
|
||||
if (unresolvedNodes.length > 0) {
|
||||
var uqSection = "\n\n=== SECTION C — UNRESOLVED OPEN QUESTIONS (NON-EVIDENCE) ===\n";
|
||||
uqSection += "The following open questions are unresolved and NOT admitted as evidence.\nThey are displayed separately to preserve epistemic boundary clarity:\n\n";
|
||||
unresolvedNodes.forEach(function(n, i) {
|
||||
uqSection += " Q" + (i + 1) + " [ID: " + n.id + "] " + (n.label || "(unnamed)") + "\n";
|
||||
if (n.description) uqSection += " " + n.description + "\n";
|
||||
});
|
||||
baselinePrompt += uqSection;
|
||||
}
|
||||
|
||||
// Use production provider path — same as synthesizeInvestigationOverview
|
||||
var provider = getProvider();
|
||||
var modelName = process.env.OLLAMA_MODEL;
|
||||
var startMs = Date.now();
|
||||
|
||||
// Reuse the overview output schema from the synthesis module
|
||||
var overviewOutputSchema = {
|
||||
type: "object",
|
||||
properties: {
|
||||
understanding: { type: "string" },
|
||||
plausibleInterpretations: { type: "string" }
|
||||
},
|
||||
required: ["understanding", "plausibleInterpretations"],
|
||||
additionalProperties: false
|
||||
};
|
||||
|
||||
var providerResult;
|
||||
try {
|
||||
providerResult = await provider.generateReconstruction(baselinePrompt, modelName, overviewOutputSchema);
|
||||
} catch (err) {
|
||||
throw new Error("Overview synthesis provider call failed: " + err.message);
|
||||
}
|
||||
|
||||
// Validate against overview-specific contract — unwrap provider envelope
|
||||
function validateOverviewResponse(raw) {
|
||||
var FORBIDDEN = ["recommendation","decision","confidenceScore","nextAction","priority","readiness"];
|
||||
if (raw == null) return { valid: false, reason: "Provider returned null/undefined" };
|
||||
var parsed;
|
||||
if (typeof raw === "string") {
|
||||
try { parsed = JSON.parse(raw); } catch(e2) { return { valid: false, reason: "Provider output is not valid JSON" }; }
|
||||
} else if (typeof raw === "object") { parsed = raw; }
|
||||
else { return { valid: false, reason: "Provider output has unexpected type" }; }
|
||||
|
||||
for (var i = 0; i < FORBIDDEN.length; i++) {
|
||||
if (FORBIDDEN[i] in parsed) return { valid: false, reason: "forbidden_field: " + FORBIDDEN[i] };
|
||||
}
|
||||
if (!parsed.understanding || typeof parsed.understanding !== "string") return { valid: false, reason: "Missing or invalid required fields" };
|
||||
if (!parsed.plausibleInterpretations || typeof parsed.plausibleInterpretations !== "string") return { valid: false, reason: "Missing or invalid required fields" };
|
||||
return { valid: true, data: parsed };
|
||||
}
|
||||
|
||||
var validated = validateOverviewResponse(providerResult.response);
|
||||
if (!validated.valid) {
|
||||
throw new Error("Overview synthesis validation failed: " + validated.reason);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
mode: "live",
|
||||
variant: "visible",
|
||||
understandingLength: validated.data.understanding.length,
|
||||
plausibleInterpretationsLength: validated.data.plausibleInterpretations.length,
|
||||
fixtureNodes: sg.nodes ? sg.nodes.length : 0,
|
||||
findingsCount: findings.length,
|
||||
unresolvedQuestionCount: unresolvedNodes.length,
|
||||
elapsedMs: Date.now() - startMs,
|
||||
overview: {
|
||||
understanding: validated.data.understanding.substring(0, 120),
|
||||
plausibleInterpretations: validated.data.plausibleInterpretations.substring(0, 120)
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
(async () => {
|
||||
var args = parseArgs(process.argv);
|
||||
@@ -80,8 +185,14 @@ if (require.main === module) {
|
||||
result.findingsNormalizedToEmptyArray;
|
||||
console.log(pass ? "PASS" : "FAIL");
|
||||
process.exit(pass ? 0 : 1);
|
||||
} else if (args.variant === "visible") {
|
||||
var result = await runVariantBExperiment(args.fixturePath);
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
process.exit(result.success ? 0 : 1);
|
||||
} else {
|
||||
// Default: hidden variant — production path (no unresolved exposure)
|
||||
var result = await runOverviewExperiment(args.fixturePath);
|
||||
result.variant = "hidden";
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
process.exit(result.success ? 0 : 1);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user