robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs(confidence-engine): record CE-ULI-11 domain-preservation intervention — explicit instruction did NOT prevent structural compression (Condition B=1 unknown, Condition C=1 unknown)

Browse Source
This commit is contained in:
robbond
2026-10-12 09:41:43 +01:00
parent 14ee66e82b
commit d5a06b9f28
4 changed files with 250 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/start-case-experiment-helper.cjs
+15 -10
View File
@@ -57,12 +57,15 @@ function readScenarioInput(argv) {
async function runStartCaseExperiment(scenarioInput, experimentInstruction, options = {}) {
const condition = options.condition;
// ── Condition B: load alt prompt + generation schema ──
// ── Condition B/C: load alt prompt + generation schema (question-free) ──
let scenarioTextForPrompt = scenarioInput?.scenario;
if (condition === "B" && typeof scenarioTextForPrompt === "string") {
if ((condition === "B" || condition === "C") && typeof scenarioTextForPrompt === "string") {
const fs = require("fs");
const path = require("path");
const altPromptPath = path.join(PROJECT_ROOT, "prompts", "reconstruct-v0.5-b-noreq.md");
const altPromptFile = condition === "B"
? "reconstruct-v0.5-b-noreq.md"
: "reconstruct-v0.5-c-domain.md";
const altPromptPath = path.join(PROJECT_ROOT, "prompts", altPromptFile);
const altPromptContent = fs.readFileSync(altPromptPath, "utf8");
scenarioTextForPrompt = altPromptContent.replace("{{SCENARIO}}", scenarioTextForPrompt);
}
@@ -80,9 +83,9 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
let reconstructionOnly = options.reconstructionOnly;
const isMock = process.env.START_CASE_EXPERIMENT_HELPER_MOCK === "1";
// Build production request — use transformed scenario text for Condition B alt prompt
const reqScenario = condition === "B" && scenarioTextForPrompt
? scenarioInput.scenario // pass raw scenario; the alt prompt wraps it
// Build production request — use raw scenario; the alt prompt wraps it
const reqScenario = (condition === "B" || condition === "C") && scenarioTextForPrompt
? scenarioInput.scenario
: (scenarioInput?.scenario);
const productionRequest = {
@@ -92,11 +95,11 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
: {}),
};
// ── Condition B schema passthrough via dependencies ──
// ── Condition B/C schema passthrough via dependencies (question-free importantUnknowns) ──
let conditionSchema = null;
if (condition === "B") {
if (condition === "B" || condition === "C") {
const schemaModule = await import(PROJECT_ROOT + "/lib/reconstruction/schema-variant.js");
conditionSchema = schemaModule.conditionBJsonSchema; // alt schema — importantUnknowns has NO question field
conditionSchema = schemaModule.conditionBJsonSchema; // question-free schema — importantUnknowns has NO question field
}
if (process.env.START_CASE_EXPERIMENT_PROVIDER === "openai") {
@@ -227,6 +230,7 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
success: result.success,
summary: result.summary ?? null,
reconstruction: result.reconstruction ?? null,
condition: condition ?? "A",
situationGraphNodeCount: result.situationGraph?.nodes?.length ?? 0,
situationGraphEdgeCount: result.situationGraph?.edges?.length ?? 0,
selectedQuestion: result.selectedQuestion?.question ?? null,
@@ -256,6 +260,7 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
const effectivePromptVersion = result.diagnostics?.promptVersion;
output.execution = {
condition: condition ?? "A",
provider: configuredProvider,
...(configuredModel !== undefined ? { model: configuredModel } : {}),
...(effectivePromptVersion !== undefined
@@ -319,7 +324,7 @@ if (require.main === module) {
experimentInstruction = process.argv[experimentIdx + 1];
}
// ── Condition selector: A (production prompt+schema) or B (alt prompt+schema) ──
// ── Condition selector: A (production) | B (question-free baseline) | C (question-free + domain-preservation) ──
const conditionArgIdx = process.argv.indexOf("--condition");
let condition = null;
if (conditionArgIdx !== -1 && conditionArgIdx + 1 < process.argv.length) {