robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(experiments): integrate Condition D/E apparatus

Browse Source
This commit is contained in:
robbond
2026-10-13 16:24:00 +01:00
parent 29cdc80f09
commit e595ee6e02
4 changed files with 234 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
prompts/reconstruct-v0.5-d-deconstruct.md
+36
View File
@@ -0,0 +1,36 @@
You are a neutral analyst performing deconstruction of the user's situation.
Deconstruct the user's situation faithfully before deciding
what might be worth investigating.
Identify the central concern and intended outcome.
Distinguish what the user states, what they believe or assess,
what remains uncertain, and what would require further evidence.
Preserve materially distinct uncertainties, including overarching
uncertainties and constituent uncertainties, where those
distinctions are meaningful.
Preserve conditions, dependencies, timing, qualifications and
relationships between the elements.
Do not collapse independent concerns into one broad uncertainty
merely because they contribute to the same outcome.
Do not create artificial uncertainties to increase their number.
Do not turn uncertainties into investigative questions.
Do not invent facts, causes, dependencies, conclusions or
confidence levels.
Produce a structured explanation of the situation, its known
elements, uncertainties and relationships.
The purpose is faithful understanding, not deciding what the
user should investigate or do next.
Scenario:
{{SCENARIO}}
prompts/reconstruct-v0.5-e-investigability-first.md
+38
View File
@@ -0,0 +1,38 @@
You are a neutral analyst performing deconstruction of the user's situation.
Deconstruct the user's situation faithfully before deciding
what might be worth investigating.
Identify the central concern and intended outcome.
Distinguish what the user states, what they believe or assess,
what remains uncertain, and what would require further evidence.
**Before considering which uncertainties matter most or how they combine, establish whether each can be investigated independently — without resolving the others.**
Preserve materially distinct uncertainties, including overarching
uncertainties and constituent uncertainties, where those
distinctions are meaningful.
Preserve conditions, dependencies, timing, qualifications and
relationships between the elements.
Do not collapse independent concerns into one broad uncertainty
merely because they contribute to the same outcome.
Do not create artificial uncertainties to increase their number.
Do not turn uncertainties into investigative questions.
Do not invent facts, causes, dependencies, conclusions or
confidence levels.
Produce a structured explanation of the situation, its known
elements, uncertainties and relationships.
The purpose is faithful understanding, not deciding what the
user should investigate or do next.
Scenario:
{{SCENARIO}}
scripts/start-case-experiment-helper.cjs
+32 -21
View File
@@ -58,9 +58,9 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
const condition = options.condition;
const treatment = options.treatment;
// ── Condition B/C/D: load alt prompt (B/C question-free; D deconstruct-freeform) ──
// ── Condition B/C/D/E: load alt prompt (B/C question-free; D deconstruct-freeform; E investigability-first freeform) ──
let scenarioTextForPrompt = scenarioInput?.scenario;
if ((condition === "B" || condition === "C" || condition === "D") && typeof scenarioTextForPrompt === "string") {
if ((condition === "B" || condition === "C" || condition === "D" || condition === "E") && typeof scenarioTextForPrompt === "string") {
const fs = require("fs");
const path = require("path");
let altPromptFile;
@@ -68,11 +68,13 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
altPromptFile = "reconstruct-v0.5-b-noreq.md";
} else if (condition === "C") {
altPromptFile = "reconstruct-v0.5-c-domain.md";
} else {
// condition === "D"
} else if (condition === "D") {
altPromptFile = treatment === "no21"
? "reconstruct-v0.5-d-deconstruct-no21.md"
: "reconstruct-v0.5-d-deconstruct.md";
} else {
// condition === "E"
altPromptFile = "reconstruct-v0.5-e-investigability-first.md";
}
const altPromptPath = path.join(PROJECT_ROOT, "prompts", altPromptFile);
const altPromptContent = fs.readFileSync(altPromptPath, "utf8");
@@ -85,8 +87,13 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
if (treatment != null && !validTreatments.includes(treatment)) {
throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21`);
}
} else if (condition === "E") {
const validTreatments = ["e-investigability-first"];
if (treatment != null && !validTreatments.includes(treatment)) {
throw new Error(`Invalid treatment: ${treatment}. Must be one of: e-investigability-first`);
}
} else if (treatment != null) {
throw new Error(`Treatment "${treatment}" is only valid with --condition D, not --condition ${condition}`);
throw new Error(`Treatment "${treatment}" is only valid with --condition D or E, not --condition ${condition}`);
}
// ── Experiment seam bridge: supply instruction to production path ──
@@ -110,13 +117,13 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
: {}),
};
// ── Condition B/C/D schema passthrough via dependencies ──
// ── Condition B/C/D/E schema passthrough via dependencies ──
let conditionSchema = null;
if (condition === "B" || condition === "C") {
const schemaModule = await import(PROJECT_ROOT + "/lib/reconstruction/schema-variant.js");
conditionSchema = schemaModule.conditionBJsonSchema; // question-free schema — importantUnknowns has NO question field
} else if (condition === "D") {
// Freeform deconstruct — explicitly no JSON structure requirement
} else if (condition === "D" || condition === "E") {
// Freeform deconstruct / investigability-first — explicitly no JSON structure requirement
conditionSchema = null;
}
@@ -137,8 +144,8 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
reconstructionOnly = true;
}
// ── Condition D: freeform mode flag ────────────────
const isFreeform = condition === "D";
// ── Condition D/E: freeform mode flag ────────────────
const isFreeform = condition === "D" || condition === "E";
if (!startCase && isMock) {
// Deterministic mode: skip environment checks and use inline test double.
@@ -277,10 +284,10 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
statusCode: result.statusCode ?? (result.success ? 200 : 500),
};
// ── Condition D: freeform deconstruct output — capture without production JSON schema ──
if (condition === "D") {
// ── Condition D/E: freeform deconstruct / investigability-first output — capture without production JSON schema ──
if (condition === "D" || condition === "E") {
const rawText = result.freeformText ?? result.reconstruction ?? null;
output.conditionType = "deconstruct";
output.conditionType = condition === "D" ? "deconstruct" : "investigability-first";
output.freeformText = rawText;
delete output.reconstruction;
}
@@ -374,12 +381,12 @@ if (require.main === module) {
}
// ── Validate condition value ──
const validConditions = ["A", "B", "C", "D"];
const validConditions = ["A", "B", "C", "D", "E"];
if (condition !== null && !validConditions.includes(condition)) {
throw new Error(`Invalid condition: ${condition}. Must be one of: A, B, C, D`);
throw new Error(`Invalid condition: ${condition}. Must be one of: A, B, C, D, E`);
}
// ── Treatment selector (D only) ──
// ── Treatment selector (D and E only) ──
const treatmentArgIdx = process.argv.indexOf("--treatment");
let treatment = null;
if (treatmentArgIdx !== -1 && treatmentArgIdx + 1 < process.argv.length) {
@@ -387,12 +394,16 @@ if (require.main === module) {
}
// ── Validate treatment value ──
const validTreatments = ["no21"];
if (treatment !== null && !validTreatments.includes(treatment)) {
const validDTreatments = ["no21"];
const validETreatments = ["e-investigability-first"];
if (treatment !== null && condition === "D" && !validDTreatments.includes(treatment)) {
throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21`);
}
if (treatment !== null && condition !== "D") {
throw new Error(`Treatment "${treatment}" is only valid with --condition D, not --condition ${condition}`);
if (treatment !== null && condition === "E" && !validETreatments.includes(treatment)) {
throw new Error(`Invalid treatment: ${treatment}. Must be one of: e-investigability-first`);
}
if (treatment !== null && !["D", "E"].includes(condition)) {
throw new Error(`Treatment "${treatment}" is only valid with --condition D or E, not --condition ${condition}`);
}
const scenarioInput = readScenarioInput(process.argv);
@@ -400,7 +411,7 @@ if (require.main === module) {
const result = await runStartCaseExperiment(scenarioInput, experimentInstruction, {
reconstructionOnly,
condition,
treatment: (condition === "D") ? treatment : undefined,
treatment: (condition === "D" || condition === "E") ? treatment : undefined,
});
console.log(JSON.stringify(result, null, 2));
tests/scripts/start-case-experiment-helper.test.js
+128 -1
View File
@@ -893,7 +893,7 @@ describe("start-case-experiment-helper.cjs apparatus", () => {
startCase: async () => ({ success: true }),
},
),
).rejects.toThrow('Treatment "no21" is only valid with --condition D, not --condition B');
).rejects.toThrow('Treatment "no21" is only valid with --condition D or E, not --condition B');
});
it("rejects unknown condition value via CLI", async () => {
@@ -984,6 +984,133 @@ describe("start-case-experiment-helper.cjs apparatus", () => {
expect(typeof originalOutput.freeformText).toBe("string");
});
});
// ── K — Condition E investigability-first apparatus ────────
describe("K — Condition E investigability-first apparatus", () => {
it("--condition E selects the investigability-first prompt rather than deconstruct", async () => {
const { runStartCaseExperiment } = require(helperPath);
let receivedScenario;
await runStartCaseExperiment(
{ scenario: "Condition E test" },
null,
{
condition: "E",
startCase: async (input) => {
receivedScenario = input.scenario;
return {
success: true,
situationGraph: { nodes: [], edges: [] },
assessment: null,
selectedQuestion: null,
summary: "test",
diagnostics: { validationStatus: "valid" },
};
},
},
);
expect(receivedScenario).toContain("You are a neutral analyst performing deconstruction of the user's situation");
expect(receivedScenario).toContain("Do not create artificial uncertainties to increase their number.");
});
it("--condition E --treatment e-investigability-first is accepted", async () => {
const { runStartCaseExperiment } = require(helperPath);
await runStartCaseExperiment(
{ scenario: "Condition E treatment test" },
null,
{
condition: "E",
treatment: "e-investigability-first",
startCase: async (input) => {
return {
success: true,
situationGraph: { nodes: [], edges: [] },
assessment: null,
selectedQuestion: null,
summary: "test",
diagnostics: { validationStatus: "valid" },
};
},
},
);
// If we reach here without an error, the treatment was accepted
});
it("--condition E produces freeformText output field via CLI mock", async () => {
const result = await runHelper(["--condition", "E", "Freeform test"]);
expect(result.exitCode).toBe(0);
const parsed = parseOutput(result);
expect(parsed).toHaveProperty("freeformText");
expect(typeof parsed.freeformText).toBe("string");
expect(parsed.conditionType).toBe("investigability-first");
});
it("--condition E sets condition to E in output", async () => {
const result = await runHelper(["--condition", "E", "Condition E flag test"]);
expect(result.exitCode).toBe(0);
const parsed = parseOutput(result);
expect(parsed.condition).toBe("E");
});
it("--condition E does not return reconstruction field in output", async () => {
const result = await runHelper(["--condition", "E", "No reconstruction test"]);
expect(result.exitCode).toBe(0);
const parsed = parseOutput(result);
expect(parsed).not.toHaveProperty("reconstruction");
});
it("--condition E --reconstruction-only sets execution.condition to E", async () => {
const result = await runHelper(["--condition", "E", "--reconstruction-only", "Reconstruction only test"]);
expect(result.exitCode).toBe(0);
const parsed = parseOutput(result);
expect(parsed.execution).toBeDefined();
expect(parsed.execution.condition).toBe("E");
});
it("--condition E rejects unknown treatment via programmatic API", async () => {
const { runStartCaseExperiment } = require(helperPath);
await expect(
runStartCaseExperiment(
{ scenario: "Invalid treatment test" },
null,
{
condition: "E",
treatment: "invalid-treatment",
startCase: async () => ({ success: true }),
},
),
).rejects.toThrow("Invalid treatment: invalid-treatment. Must be one of: e-investigability-first");
});
it("--condition E rejects treatment with non-E condition via programmatic API", async () => {
const { runStartCaseExperiment } = require(helperPath);
await expect(
runStartCaseExperiment(
{ scenario: "Cross-condition test" },
null,
{
condition: "B",
treatment: "e-investigability-first",
startCase: async () => ({ success: true }),
},
),
).rejects.toThrow('Treatment "e-investigability-first" is only valid with --condition D or E, not --condition B');
});
it("--condition E exposes non-empty freeformText (mock)", async () => {
const result = await runHelper(["--condition", "E", "Freeform content test"]);
expect(result.exitCode).toBe(0);
const parsed = parseOutput(result);
expect(parsed.freeformText.length).toBeGreaterThan(0);
});
});
});
// ── Evidence persistence wrapper tests ─────────────────────────────