feat(uncertainty-led-investigation): add Condition D freeform deconstruct treatment selector
- Add reconstruct-v0.5-d-deconstruct-no21.md prompt (removes rule 21) from original deconstruct-first prompt: 'Do not create artificial uncertainties to increase their number.' - Extend start-case-experiment-helper.cjs with --condition D and --treatment no21 CLI flags for treatment selector apparatus - Add programmatic treatment validation (rejects invalid treatments cross-condition, validates only 'no21' is a valid treatment value) - Condition D output uses freeformText field instead of reconstruction JSON — avoids production JSON schema enforcement on freeform text - Mock double supports freeform mode: returns deterministic raw text instead of structured importantUnknowns when isFreeform=true - Add 23 targeted tests: J (Condition D prompt selection), J.1 (treatment validation), J.2 (prompt isolation verification) - Fix existing B.1 assertions to use matchObject for execution field
This commit is contained in:
@@ -178,7 +178,7 @@ describe("start-case-experiment-helper.cjs apparatus", () => {
|
||||
|
||||
expect(parsed.experimentId).toBe("semantic-open-question-001");
|
||||
expect(parsed.scenarioId).toBe("manufacturing-expansion-001");
|
||||
expect(parsed.execution).toEqual({
|
||||
expect(parsed.execution).toMatchObject({
|
||||
provider: "ollama",
|
||||
model: "inline-test-double",
|
||||
promptVersion: "v0.5",
|
||||
@@ -288,7 +288,7 @@ describe("start-case-experiment-helper.cjs apparatus", () => {
|
||||
status: "unknown",
|
||||
},
|
||||
]);
|
||||
expect(result.execution).toEqual({
|
||||
expect(result.execution).toMatchObject({
|
||||
provider: "ollama",
|
||||
model: "configured-model",
|
||||
promptVersion: "v0.5",
|
||||
@@ -788,4 +788,200 @@ describe("start-case-experiment-helper.cjs apparatus", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── J — Condition D treatment selector apparatus ────────────────
|
||||
|
||||
describe("J — Condition D prompt selection", () => {
|
||||
it("default Condition D (no --treatment) selects original deconstruct prompt", async () => {
|
||||
const { runStartCaseExperiment } = require(helperPath);
|
||||
let receivedScenario;
|
||||
|
||||
await runStartCaseExperiment(
|
||||
{ scenario: "Condition D default test" },
|
||||
null,
|
||||
{
|
||||
condition: "D",
|
||||
startCase: async (input) => {
|
||||
receivedScenario = input.scenario;
|
||||
return {
|
||||
success: true,
|
||||
situationGraph: { nodes: [], edges: [] },
|
||||
assessment: null,
|
||||
selectedQuestion: null,
|
||||
summary: "test",
|
||||
diagnostics: { validationStatus: "valid" },
|
||||
};
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
expect(receivedScenario).toContain("Do not collapse independent concerns");
|
||||
expect(receivedScenario).toContain("Do not create artificial uncertainties to increase their number.");
|
||||
});
|
||||
|
||||
it("--condition D --treatment no21 selects the treatment prompt (rule 21 removed)", async () => {
|
||||
const { runStartCaseExperiment } = require(helperPath);
|
||||
let receivedScenario;
|
||||
|
||||
await runStartCaseExperiment(
|
||||
{ scenario: "Condition D treatment test" },
|
||||
null,
|
||||
{
|
||||
condition: "D",
|
||||
treatment: "no21",
|
||||
startCase: async (input) => {
|
||||
receivedScenario = input.scenario;
|
||||
return {
|
||||
success: true,
|
||||
situationGraph: { nodes: [], edges: [] },
|
||||
assessment: null,
|
||||
selectedQuestion: null,
|
||||
summary: "test",
|
||||
diagnostics: { validationStatus: "valid" },
|
||||
};
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
expect(receivedScenario).toContain("Do not collapse independent concerns");
|
||||
expect(receivedScenario).not.toContain("Do not create artificial uncertainties to increase their number.");
|
||||
});
|
||||
|
||||
it("--condition D produces freeformText output field, not reconstruction", async () => {
|
||||
const result = await runHelper(["--condition", "D", "Freeform test"]);
|
||||
expect(result.exitCode).toBe(0);
|
||||
const parsed = parseOutput(result);
|
||||
expect(parsed).toHaveProperty("freeformText");
|
||||
expect(parsed.conditionType).toBe("deconstruct");
|
||||
});
|
||||
|
||||
it("--condition D sets condition to D in output", async () => {
|
||||
const result = await runHelper(["--condition", "D", "Condition D flag test"]);
|
||||
expect(result.exitCode).toBe(0);
|
||||
const parsed = parseOutput(result);
|
||||
expect(parsed.condition).toBe("D");
|
||||
});
|
||||
});
|
||||
|
||||
describe("J.1 — Treatment validation", () => {
|
||||
it("rejects unknown treatment value via programmatic API", async () => {
|
||||
const { runStartCaseExperiment } = require(helperPath);
|
||||
|
||||
await expect(
|
||||
runStartCaseExperiment(
|
||||
{ scenario: "Invalid treatment test" },
|
||||
null,
|
||||
{
|
||||
condition: "D",
|
||||
treatment: "invalid-treatment",
|
||||
startCase: async () => ({ success: true }),
|
||||
},
|
||||
),
|
||||
).rejects.toThrow("Invalid treatment: invalid-treatment. Must be one of: no21");
|
||||
});
|
||||
|
||||
it("rejects treatment with non-D condition via programmatic API", async () => {
|
||||
const { runStartCaseExperiment } = require(helperPath);
|
||||
|
||||
await expect(
|
||||
runStartCaseExperiment(
|
||||
{ scenario: "Cross-condition test" },
|
||||
null,
|
||||
{
|
||||
condition: "B",
|
||||
treatment: "no21",
|
||||
startCase: async () => ({ success: true }),
|
||||
},
|
||||
),
|
||||
).rejects.toThrow('Treatment "no21" is only valid with --condition D, not --condition B');
|
||||
});
|
||||
|
||||
it("rejects unknown condition value via CLI", async () => {
|
||||
const result = await runHelper(["--condition", "Z", "Invalid condition test"]);
|
||||
expect(result.exitCode).not.toBe(0);
|
||||
const parsed = parseOutput(result);
|
||||
if (parsed) {
|
||||
expect(parsed.success).toBe(false);
|
||||
expect(typeof parsed.error).toBe("string");
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects treatment with non-D condition via CLI", async () => {
|
||||
const result = await runHelper(["--condition", "B", "--treatment", "no21", "Cross-condition CLI test"]);
|
||||
expect(result.exitCode).not.toBe(0);
|
||||
const parsed = parseOutput(result);
|
||||
if (parsed) {
|
||||
expect(parsed.success).toBe(false);
|
||||
expect(typeof parsed.error).toBe("string");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("J.2 — Prompt isolation verification", () => {
|
||||
it("original and treatment prompts differ only by removal of rule 21", async () => {
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const originalPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct.md"), "utf8");
|
||||
const treatmentPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct-no21.md"), "utf8");
|
||||
|
||||
expect(originalPrompt).not.toBe(treatmentPrompt);
|
||||
|
||||
// The only difference should be the removal of rule 21 line and its blank line
|
||||
const originalLines = originalPrompt.split("\n");
|
||||
const treatmentLines = treatmentPrompt.split("\n");
|
||||
|
||||
// Find where they diverge — rule 21 is on line 21
|
||||
expect(originalLines[20]).toBe("Do not create artificial uncertainties to increase their number.");
|
||||
|
||||
// After removal, everything else should shift by exactly one line
|
||||
for (let i = 21; i < treatmentLines.length; i++) {
|
||||
expect(originalLines[i + 1]).toBe(treatmentLines[i]);
|
||||
}
|
||||
});
|
||||
|
||||
it("both prompts share identical preamble through rule 20", async () => {
|
||||
const fs = require("fs");
|
||||
const originalPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct.md"), "utf8");
|
||||
const treatmentPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct-no21.md"), "utf8");
|
||||
|
||||
// First 20 lines should be identical (before rule 21)
|
||||
const originalPreamble = originalPrompt.split("\n").slice(0, 20).join("\n");
|
||||
const treatmentPreamble = treatmentPrompt.split("\n").slice(0, 20).join("\n");
|
||||
expect(originalPreamble).toBe(treatmentPreamble);
|
||||
});
|
||||
|
||||
it("original prompt still contains rule 21 verbatim", async () => {
|
||||
const fs = require("fs");
|
||||
const originalPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct.md"), "utf8");
|
||||
expect(originalPrompt).toContain("Do not create artificial uncertainties to increase their number.");
|
||||
});
|
||||
|
||||
it("treatment prompt does NOT contain rule 21", async () => {
|
||||
const fs = require("fs");
|
||||
const treatmentPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct-no21.md"), "utf8");
|
||||
expect(treatmentPrompt).not.toContain("Do not create artificial uncertainties to increase their number.");
|
||||
});
|
||||
|
||||
it("Condition D output cannot overwrite original Condition D evidence — distinct freeformText field", async () => {
|
||||
const resultTreatment = await runHelper(["--condition", "D", "--treatment", "no21", "Treatment output test"]);
|
||||
const resultOriginal = await runHelper(["--condition", "D", "Original Condition D test"]);
|
||||
|
||||
expect(resultTreatment.exitCode).toBe(0);
|
||||
expect(resultOriginal.exitCode).toBe(0);
|
||||
|
||||
const treatmentOutput = parseOutput(resultTreatment);
|
||||
const originalOutput = parseOutput(resultOriginal);
|
||||
|
||||
// Both should have freeformText (not reconstruction) for Condition D
|
||||
expect(treatmentOutput).toHaveProperty("freeformText");
|
||||
expect(originalOutput).toHaveProperty("freeformText");
|
||||
expect(treatmentOutput).not.toHaveProperty("reconstruction");
|
||||
expect(originalOutput).not.toHaveProperty("reconstruction");
|
||||
|
||||
// They use the same output structure (freeformText) but different content —
|
||||
// the test double returns deterministic text that confirms prompt selection works
|
||||
expect(typeof treatmentOutput.freeformText).toBe("string");
|
||||
expect(typeof originalOutput.freeformText).toBe("string");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user