robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(uncertainty-led-investigation): add Condition D freeform deconstruct treatment selector

Browse Source 
- Add reconstruct-v0.5-d-deconstruct-no21.md prompt (removes rule 21)
  from original deconstruct-first prompt: 'Do not create artificial
  uncertainties to increase their number.'
- Extend start-case-experiment-helper.cjs with --condition D and
  --treatment no21 CLI flags for treatment selector apparatus
- Add programmatic treatment validation (rejects invalid treatments
  cross-condition, validates only 'no21' is a valid treatment value)
- Condition D output uses freeformText field instead of reconstruction
  JSON — avoids production JSON schema enforcement on freeform text
- Mock double supports freeform mode: returns deterministic raw text
  instead of structured importantUnknowns when isFreeform=true
- Add 23 targeted tests: J (Condition D prompt selection),
  J.1 (treatment validation), J.2 (prompt isolation verification)
- Fix existing B.1 assertions to use matchObject for execution field
This commit is contained in:
robbond
2026-10-12 18:45:53 +01:00
parent d5a06b9f28
commit 5af3e62ddd
3 changed files with 321 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/start-case-experiment-helper.cjs
+88 -23
View File
@@ -56,20 +56,39 @@ function readScenarioInput(argv) {
async function runStartCaseExperiment(scenarioInput, experimentInstruction, options = {}) {
const condition = options.condition;
const treatment = options.treatment;
// ── Condition B/C: load alt prompt + generation schema (question-free) ──
// ── Condition B/C/D: load alt prompt (B/C question-free; D deconstruct-freeform) ──
let scenarioTextForPrompt = scenarioInput?.scenario;
if ((condition === "B" || condition === "C") && typeof scenarioTextForPrompt === "string") {
if ((condition === "B" || condition === "C" || condition === "D") && typeof scenarioTextForPrompt === "string") {
const fs = require("fs");
const path = require("path");
const altPromptFile = condition === "B"
? "reconstruct-v0.5-b-noreq.md"
: "reconstruct-v0.5-c-domain.md";
let altPromptFile;
if (condition === "B") {
altPromptFile = "reconstruct-v0.5-b-noreq.md";
} else if (condition === "C") {
altPromptFile = "reconstruct-v0.5-c-domain.md";
} else {
// condition === "D"
altPromptFile = treatment === "no21"
? "reconstruct-v0.5-d-deconstruct-no21.md"
: "reconstruct-v0.5-d-deconstruct.md";
}
const altPromptPath = path.join(PROJECT_ROOT, "prompts", altPromptFile);
const altPromptContent = fs.readFileSync(altPromptPath, "utf8");
scenarioTextForPrompt = altPromptContent.replace("{{SCENARIO}}", scenarioTextForPrompt);
}
// ── Treatment validation (programmatic API) ────────────────
if (condition === "D") {
const validTreatments = ["no21"];
if (treatment != null && !validTreatments.includes(treatment)) {
throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21`);
}
} else if (treatment != null) {
throw new Error(`Treatment "${treatment}" is only valid with --condition D, not --condition ${condition}`);
}
// ── Experiment seam bridge: supply instruction to production path ──
const hadEnvVar = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION != null;
const previousEnvValue = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION;
@@ -83,23 +102,22 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
let reconstructionOnly = options.reconstructionOnly;
const isMock = process.env.START_CASE_EXPERIMENT_HELPER_MOCK === "1";
// Build production request — use raw scenario; the alt prompt wraps it
const reqScenario = (condition === "B" || condition === "C") && scenarioTextForPrompt
? scenarioInput.scenario
: (scenarioInput?.scenario);
// Build production request — use prepared alt prompt or raw scenario
const productionRequest = {
scenario: reqScenario,
scenario: scenarioTextForPrompt ?? scenarioInput?.scenario,
...(scenarioInput?.promptVersion !== undefined
? { promptVersion: scenarioInput.promptVersion }
: {}),
};
// ── Condition B/C schema passthrough via dependencies (question-free importantUnknowns) ──
// ── Condition B/C/D schema passthrough via dependencies ──
let conditionSchema = null;
if (condition === "B" || condition === "C") {
const schemaModule = await import(PROJECT_ROOT + "/lib/reconstruction/schema-variant.js");
conditionSchema = schemaModule.conditionBJsonSchema; // question-free schema — importantUnknowns has NO question field
} else if (condition === "D") {
// Freeform deconstruct — explicitly no JSON structure requirement
conditionSchema = null;
}
if (process.env.START_CASE_EXPERIMENT_PROVIDER === "openai") {
@@ -119,6 +137,9 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
reconstructionOnly = true;
}
// ── Condition D: freeform mode flag ────────────────
const isFreeform = condition === "D";
if (!startCase && isMock) {
// Deterministic mode: skip environment checks and use inline test double.
startCase = async (body, dependencies) => {
@@ -130,18 +151,30 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
if (shouldFail) {
return { success: false, error: "deterministic mock failure", statusCode: 500 };
}
// In freeform mode (Condition D), the model returns raw text rather than structured data.
const reconstructionResult = isFreeform
? null
: {
importantUnknowns: [
{
id: "unknown-1",
description: "Canonical uncertainty from the deterministic test double",
confidence: "medium",
question: "What would resolve the deterministic uncertainty?",
},
],
};
// In freeform mode, provide deterministic raw text for test apparatus verification.
const freeformText = isFreeform
? "Deterministic freeform deconstruct result from inline test double — no live model calls."
: null;
return {
success: true,
reconstruction: {
importantUnknowns: [
{
id: "unknown-1",
description: "Canonical uncertainty from the deterministic test double",
confidence: "medium",
question: "What would resolve the deterministic uncertainty?",
},
],
},
reconstruction: reconstructionResult,
freeformText,
situationGraph: {
nodes: [
{
@@ -204,6 +237,7 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
reconstructionProvider,
reconstructionModelName,
reconstructionOnly,
...(isFreeform ? { freeform: true } : {}),
};
if (conditionSchema) {
startCaseDeps.generationSchema = conditionSchema;
@@ -243,6 +277,14 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
statusCode: result.statusCode ?? (result.success ? 200 : 500),
};
// ── Condition D: freeform deconstruct output — capture without production JSON schema ──
if (condition === "D") {
const rawText = result.freeformText ?? result.reconstruction ?? null;
output.conditionType = "deconstruct";
output.freeformText = rawText;
delete output.reconstruction;
}
if (reconstructionOnly) {
if (scenarioInput?.experimentId !== undefined) {
output.experimentId = scenarioInput.experimentId;
@@ -324,18 +366,41 @@ if (require.main === module) {
experimentInstruction = process.argv[experimentIdx + 1];
}
// ── Condition selector: A (production) | B (question-free baseline) | C (question-free + domain-preservation) ──
// ── Condition selector: A (production) | B (question-free baseline) | C (question-free + domain-preservation) | D (deconstruct-first freeform) ──
const conditionArgIdx = process.argv.indexOf("--condition");
let condition = null;
if (conditionArgIdx !== -1 && conditionArgIdx + 1 < process.argv.length) {
condition = process.argv[conditionArgIdx + 1];
}
// ── Validate condition value ──
const validConditions = ["A", "B", "C", "D"];
if (condition !== null && !validConditions.includes(condition)) {
throw new Error(`Invalid condition: ${condition}. Must be one of: A, B, C, D`);
}
// ── Treatment selector (D only) ──
const treatmentArgIdx = process.argv.indexOf("--treatment");
let treatment = null;
if (treatmentArgIdx !== -1 && treatmentArgIdx + 1 < process.argv.length) {
treatment = process.argv[treatmentArgIdx + 1];
}
// ── Validate treatment value ──
const validTreatments = ["no21"];
if (treatment !== null && !validTreatments.includes(treatment)) {
throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21`);
}
if (treatment !== null && condition !== "D") {
throw new Error(`Treatment "${treatment}" is only valid with --condition D, not --condition ${condition}`);
}
const scenarioInput = readScenarioInput(process.argv);
const reconstructionOnly = process.argv.includes("--reconstruction-only");
const result = await runStartCaseExperiment(scenarioInput, experimentInstruction, {
reconstructionOnly,
condition,
treatment: (condition === "D") ? treatment : undefined,
});
console.log(JSON.stringify(result, null, 2));