feat(confidence-engine): add CE-ULI-08 two-condition uncertainty comparison apparatus
Adds experimental apparatus for testing whether removing the question field from importantUnknowns in reconstruction output improves semantic fidelity. New files: - lib/reconstruction/schema-variant.js (conditionB schema + production passthrough) - prompts/reconstruct-v0.5-b-noreq.md (alt prompt, identical line count to prod) Changed files: - lib/graph/orchestrator.js (pass generationSchema through dependencies) - lib/analysis.js (accept opts.generationSchema fallback to original schema) - scripts/start-case-experiment-helper.cjs (--condition B/A flag support) - tests/scripts/start-case-experiment-helper.test.js (8 new deterministic tests) 43 targeted deterministic tests PASS. Production callers use original prompt/schema unchanged. No live model calls. No full suite evidence available. Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -55,6 +55,18 @@ function readScenarioInput(argv) {
|
||||
}
|
||||
|
||||
async function runStartCaseExperiment(scenarioInput, experimentInstruction, options = {}) {
|
||||
const condition = options.condition;
|
||||
|
||||
// ── Condition B: load alt prompt + generation schema ──
|
||||
let scenarioTextForPrompt = scenarioInput?.scenario;
|
||||
if (condition === "B" && typeof scenarioTextForPrompt === "string") {
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const altPromptPath = path.join(PROJECT_ROOT, "prompts", "reconstruct-v0.5-b-noreq.md");
|
||||
const altPromptContent = fs.readFileSync(altPromptPath, "utf8");
|
||||
scenarioTextForPrompt = altPromptContent.replace("{{SCENARIO}}", scenarioTextForPrompt);
|
||||
}
|
||||
|
||||
// ── Experiment seam bridge: supply instruction to production path ──
|
||||
const hadEnvVar = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION != null;
|
||||
const previousEnvValue = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION;
|
||||
@@ -67,13 +79,26 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
|
||||
let reconstructionModelName = options.reconstructionModelName;
|
||||
let reconstructionOnly = options.reconstructionOnly;
|
||||
const isMock = process.env.START_CASE_EXPERIMENT_HELPER_MOCK === "1";
|
||||
|
||||
// Build production request — use transformed scenario text for Condition B alt prompt
|
||||
const reqScenario = condition === "B" && scenarioTextForPrompt
|
||||
? scenarioInput.scenario // pass raw scenario; the alt prompt wraps it
|
||||
: (scenarioInput?.scenario);
|
||||
|
||||
const productionRequest = {
|
||||
scenario: scenarioInput?.scenario,
|
||||
scenario: reqScenario,
|
||||
...(scenarioInput?.promptVersion !== undefined
|
||||
? { promptVersion: scenarioInput.promptVersion }
|
||||
: {}),
|
||||
};
|
||||
|
||||
// ── Condition B schema passthrough via dependencies ──
|
||||
let conditionSchema = null;
|
||||
if (condition === "B") {
|
||||
const schemaModule = await import(PROJECT_ROOT + "/lib/reconstruction/schema-variant.js");
|
||||
conditionSchema = schemaModule.conditionBJsonSchema; // alt schema — importantUnknowns has NO question field
|
||||
}
|
||||
|
||||
if (process.env.START_CASE_EXPERIMENT_PROVIDER === "openai") {
|
||||
if (!process.env.OPENAI_API_KEY) {
|
||||
throw new Error(
|
||||
@@ -93,7 +118,10 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
|
||||
|
||||
if (!startCase && isMock) {
|
||||
// Deterministic mode: skip environment checks and use inline test double.
|
||||
startCase = async (_body, dependencies) => {
|
||||
startCase = async (body, dependencies) => {
|
||||
// Capture the generation schema used for verification
|
||||
conditionSchema = dependencies.generationSchema ?? null;
|
||||
|
||||
// Deterministic inline test double — no live model calls.
|
||||
const shouldFail = process.env.START_CASE_EXPERIMENT_HELPER_FORCE_FAIL === "1";
|
||||
if (shouldFail) {
|
||||
@@ -169,11 +197,15 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
|
||||
let result;
|
||||
const endToEndStartedAt = Date.now();
|
||||
try {
|
||||
result = await startCase(productionRequest, {
|
||||
const startCaseDeps = {
|
||||
reconstructionProvider,
|
||||
reconstructionModelName,
|
||||
reconstructionOnly,
|
||||
});
|
||||
};
|
||||
if (conditionSchema) {
|
||||
startCaseDeps.generationSchema = conditionSchema;
|
||||
}
|
||||
result = await startCase(productionRequest, startCaseDeps);
|
||||
} finally {
|
||||
// Clean up experiment env var after execution regardless of outcome
|
||||
if (experimentInstruction && !hadEnvVar) {
|
||||
@@ -287,10 +319,18 @@ if (require.main === module) {
|
||||
experimentInstruction = process.argv[experimentIdx + 1];
|
||||
}
|
||||
|
||||
// ── Condition selector: A (production prompt+schema) or B (alt prompt+schema) ──
|
||||
const conditionArgIdx = process.argv.indexOf("--condition");
|
||||
let condition = null;
|
||||
if (conditionArgIdx !== -1 && conditionArgIdx + 1 < process.argv.length) {
|
||||
condition = process.argv[conditionArgIdx + 1];
|
||||
}
|
||||
|
||||
const scenarioInput = readScenarioInput(process.argv);
|
||||
const reconstructionOnly = process.argv.includes("--reconstruction-only");
|
||||
const result = await runStartCaseExperiment(scenarioInput, experimentInstruction, {
|
||||
reconstructionOnly,
|
||||
condition,
|
||||
});
|
||||
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
|
||||
Reference in New Issue
Block a user