robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(experiments): integrate Condition D/E apparatus

Browse Source
This commit is contained in:
robbond
2026-10-13 16:24:00 +01:00
parent 29cdc80f09
commit e595ee6e02
4 changed files with 234 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/start-case-experiment-helper.cjs
+32 -21
View File
@@ -58,9 +58,9 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
const condition = options.condition;
const treatment = options.treatment;
// ── Condition B/C/D: load alt prompt (B/C question-free; D deconstruct-freeform) ──
// ── Condition B/C/D/E: load alt prompt (B/C question-free; D deconstruct-freeform; E investigability-first freeform) ──
let scenarioTextForPrompt = scenarioInput?.scenario;
if ((condition === "B" || condition === "C" || condition === "D") && typeof scenarioTextForPrompt === "string") {
if ((condition === "B" || condition === "C" || condition === "D" || condition === "E") && typeof scenarioTextForPrompt === "string") {
const fs = require("fs");
const path = require("path");
let altPromptFile;
@@ -68,11 +68,13 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
altPromptFile = "reconstruct-v0.5-b-noreq.md";
} else if (condition === "C") {
altPromptFile = "reconstruct-v0.5-c-domain.md";
} else {
// condition === "D"
} else if (condition === "D") {
altPromptFile = treatment === "no21"
? "reconstruct-v0.5-d-deconstruct-no21.md"
: "reconstruct-v0.5-d-deconstruct.md";
} else {
// condition === "E"
altPromptFile = "reconstruct-v0.5-e-investigability-first.md";
}
const altPromptPath = path.join(PROJECT_ROOT, "prompts", altPromptFile);
const altPromptContent = fs.readFileSync(altPromptPath, "utf8");
@@ -85,8 +87,13 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
if (treatment != null && !validTreatments.includes(treatment)) {
throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21`);
}
} else if (condition === "E") {
const validTreatments = ["e-investigability-first"];
if (treatment != null && !validTreatments.includes(treatment)) {
throw new Error(`Invalid treatment: ${treatment}. Must be one of: e-investigability-first`);
}
} else if (treatment != null) {
throw new Error(`Treatment "${treatment}" is only valid with --condition D, not --condition ${condition}`);
throw new Error(`Treatment "${treatment}" is only valid with --condition D or E, not --condition ${condition}`);
}
// ── Experiment seam bridge: supply instruction to production path ──
@@ -110,13 +117,13 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
: {}),
};
// ── Condition B/C/D schema passthrough via dependencies ──
// ── Condition B/C/D/E schema passthrough via dependencies ──
let conditionSchema = null;
if (condition === "B" || condition === "C") {
const schemaModule = await import(PROJECT_ROOT + "/lib/reconstruction/schema-variant.js");
conditionSchema = schemaModule.conditionBJsonSchema; // question-free schema — importantUnknowns has NO question field
} else if (condition === "D") {
// Freeform deconstruct — explicitly no JSON structure requirement
} else if (condition === "D" || condition === "E") {
// Freeform deconstruct / investigability-first — explicitly no JSON structure requirement
conditionSchema = null;
}
@@ -137,8 +144,8 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
reconstructionOnly = true;
}
// ── Condition D: freeform mode flag ────────────────
const isFreeform = condition === "D";
// ── Condition D/E: freeform mode flag ────────────────
const isFreeform = condition === "D" || condition === "E";
if (!startCase && isMock) {
// Deterministic mode: skip environment checks and use inline test double.
@@ -277,10 +284,10 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
statusCode: result.statusCode ?? (result.success ? 200 : 500),
};
// ── Condition D: freeform deconstruct output — capture without production JSON schema ──
if (condition === "D") {
// ── Condition D/E: freeform deconstruct / investigability-first output — capture without production JSON schema ──
if (condition === "D" || condition === "E") {
const rawText = result.freeformText ?? result.reconstruction ?? null;
output.conditionType = "deconstruct";
output.conditionType = condition === "D" ? "deconstruct" : "investigability-first";
output.freeformText = rawText;
delete output.reconstruction;
}
@@ -374,12 +381,12 @@ if (require.main === module) {
}
// ── Validate condition value ──
const validConditions = ["A", "B", "C", "D"];
const validConditions = ["A", "B", "C", "D", "E"];
if (condition !== null && !validConditions.includes(condition)) {
throw new Error(`Invalid condition: ${condition}. Must be one of: A, B, C, D`);
throw new Error(`Invalid condition: ${condition}. Must be one of: A, B, C, D, E`);
}
// ── Treatment selector (D only) ──
// ── Treatment selector (D and E only) ──
const treatmentArgIdx = process.argv.indexOf("--treatment");
let treatment = null;
if (treatmentArgIdx !== -1 && treatmentArgIdx + 1 < process.argv.length) {
@@ -387,12 +394,16 @@ if (require.main === module) {
}
// ── Validate treatment value ──
const validTreatments = ["no21"];
if (treatment !== null && !validTreatments.includes(treatment)) {
const validDTreatments = ["no21"];
const validETreatments = ["e-investigability-first"];
if (treatment !== null && condition === "D" && !validDTreatments.includes(treatment)) {
throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21`);
}
if (treatment !== null && condition !== "D") {
throw new Error(`Treatment "${treatment}" is only valid with --condition D, not --condition ${condition}`);
if (treatment !== null && condition === "E" && !validETreatments.includes(treatment)) {
throw new Error(`Invalid treatment: ${treatment}. Must be one of: e-investigability-first`);
}
if (treatment !== null && !["D", "E"].includes(condition)) {
throw new Error(`Treatment "${treatment}" is only valid with --condition D or E, not --condition ${condition}`);
}
const scenarioInput = readScenarioInput(process.argv);
@@ -400,7 +411,7 @@ if (require.main === module) {
const result = await runStartCaseExperiment(scenarioInput, experimentInstruction, {
reconstructionOnly,
condition,
treatment: (condition === "D") ? treatment : undefined,
treatment: (condition === "D" || condition === "E") ? treatment : undefined,
});
console.log(JSON.stringify(result, null, 2));