feat(confidence-engine): add CE-ULI-08 two-condition uncertainty comparison apparatus

Adds experimental apparatus for testing whether removing the question field
from importantUnknowns in reconstruction output improves semantic fidelity.

New files:
- lib/reconstruction/schema-variant.js (conditionB schema + production passthrough)
- prompts/reconstruct-v0.5-b-noreq.md (alt prompt, identical line count to prod)

Changed files:
- lib/graph/orchestrator.js (pass generationSchema through dependencies)
- lib/analysis.js (accept opts.generationSchema fallback to original schema)
- scripts/start-case-experiment-helper.cjs (--condition B/A flag support)
- tests/scripts/start-case-experiment-helper.test.js (8 new deterministic tests)

43 targeted deterministic tests PASS. Production callers use original prompt/schema unchanged.
No live model calls. No full suite evidence available.

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
2026-10-12 07:33:31 +01:00
co-authored by Claude Code
parent a54b36a2b2
commit 3b04d05355
7 changed files with 544 additions and 5 deletions
+44 -4
View File
@@ -55,6 +55,18 @@ function readScenarioInput(argv) {
}
async function runStartCaseExperiment(scenarioInput, experimentInstruction, options = {}) {
const condition = options.condition;
// ── Condition B: load alt prompt + generation schema ──
let scenarioTextForPrompt = scenarioInput?.scenario;
if (condition === "B" && typeof scenarioTextForPrompt === "string") {
const fs = require("fs");
const path = require("path");
const altPromptPath = path.join(PROJECT_ROOT, "prompts", "reconstruct-v0.5-b-noreq.md");
const altPromptContent = fs.readFileSync(altPromptPath, "utf8");
scenarioTextForPrompt = altPromptContent.replace("{{SCENARIO}}", scenarioTextForPrompt);
}
// ── Experiment seam bridge: supply instruction to production path ──
const hadEnvVar = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION != null;
const previousEnvValue = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION;
@@ -67,13 +79,26 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
let reconstructionModelName = options.reconstructionModelName;
let reconstructionOnly = options.reconstructionOnly;
const isMock = process.env.START_CASE_EXPERIMENT_HELPER_MOCK === "1";
// Build production request — use transformed scenario text for Condition B alt prompt
const reqScenario = condition === "B" && scenarioTextForPrompt
? scenarioInput.scenario // pass raw scenario; the alt prompt wraps it
: (scenarioInput?.scenario);
const productionRequest = {
scenario: scenarioInput?.scenario,
scenario: reqScenario,
...(scenarioInput?.promptVersion !== undefined
? { promptVersion: scenarioInput.promptVersion }
: {}),
};
// ── Condition B schema passthrough via dependencies ──
let conditionSchema = null;
if (condition === "B") {
const schemaModule = await import(PROJECT_ROOT + "/lib/reconstruction/schema-variant.js");
conditionSchema = schemaModule.conditionBJsonSchema; // alt schema — importantUnknowns has NO question field
}
if (process.env.START_CASE_EXPERIMENT_PROVIDER === "openai") {
if (!process.env.OPENAI_API_KEY) {
throw new Error(
@@ -93,7 +118,10 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
if (!startCase && isMock) {
// Deterministic mode: skip environment checks and use inline test double.
startCase = async (_body, dependencies) => {
startCase = async (body, dependencies) => {
// Capture the generation schema used for verification
conditionSchema = dependencies.generationSchema ?? null;
// Deterministic inline test double — no live model calls.
const shouldFail = process.env.START_CASE_EXPERIMENT_HELPER_FORCE_FAIL === "1";
if (shouldFail) {
@@ -169,11 +197,15 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
let result;
const endToEndStartedAt = Date.now();
try {
result = await startCase(productionRequest, {
const startCaseDeps = {
reconstructionProvider,
reconstructionModelName,
reconstructionOnly,
});
};
if (conditionSchema) {
startCaseDeps.generationSchema = conditionSchema;
}
result = await startCase(productionRequest, startCaseDeps);
} finally {
// Clean up experiment env var after execution regardless of outcome
if (experimentInstruction && !hadEnvVar) {
@@ -287,10 +319,18 @@ if (require.main === module) {
experimentInstruction = process.argv[experimentIdx + 1];
}
// ── Condition selector: A (production prompt+schema) or B (alt prompt+schema) ──
const conditionArgIdx = process.argv.indexOf("--condition");
let condition = null;
if (conditionArgIdx !== -1 && conditionArgIdx + 1 < process.argv.length) {
condition = process.argv[conditionArgIdx + 1];
}
const scenarioInput = readScenarioInput(process.argv);
const reconstructionOnly = process.argv.includes("--reconstruction-only");
const result = await runStartCaseExperiment(scenarioInput, experimentInstruction, {
reconstructionOnly,
condition,
});
console.log(JSON.stringify(result, null, 2));