feat(confidence-engine): add CE-ULI-08 two-condition uncertainty comparison apparatus

Adds experimental apparatus for testing whether removing the question field
from importantUnknowns in reconstruction output improves semantic fidelity.

New files:
- lib/reconstruction/schema-variant.js (conditionB schema + production passthrough)
- prompts/reconstruct-v0.5-b-noreq.md (alt prompt, identical line count to prod)

Changed files:
- lib/graph/orchestrator.js (pass generationSchema through dependencies)
- lib/analysis.js (accept opts.generationSchema fallback to original schema)
- scripts/start-case-experiment-helper.cjs (--condition B/A flag support)
- tests/scripts/start-case-experiment-helper.test.js (8 new deterministic tests)

43 targeted deterministic tests PASS. Production callers use original prompt/schema unchanged.
No live model calls. No full suite evidence available.

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
2026-10-12 07:33:31 +01:00
co-authored by Claude Code
parent a54b36a2b2
commit 3b04d05355
7 changed files with 544 additions and 5 deletions
@@ -654,4 +654,138 @@ describe("start-case-experiment-helper.cjs apparatus", () => {
expect(typeof parsed.failureReason).toBe("string");
});
});
// ── I — Condition B schema variant apparatus ────────────────────
describe("I — Condition B schema variant apparatus", () => {
it("--condition B passes generationSchema through the startCase seam", async () => {
const { runStartCaseExperiment } = require(helperPath);
let receivedDependencies;
const result = await runStartCaseExperiment(
{ scenario: "Condition B test scenario" },
null,
{
condition: "B",
startCase: async (_input, dependencies) => {
receivedDependencies = dependencies;
return {
success: true,
situationGraph: { nodes: [], edges: [] },
assessment: null,
selectedQuestion: null,
summary: "test",
diagnostics: { validationStatus: "valid" },
};
},
},
);
expect(result.success).toBe(true);
expect(receivedDependencies.generationSchema).toBeDefined();
// Condition B schema: importantUnknowns items should NOT have question property
const iuSchema = receivedDependencies.generationSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items;
if (iuSchema && iuSchema.properties) {
expect(iuSchema.properties).not.toHaveProperty("question");
}
});
it("--condition A uses production schema with question required", async () => {
const { runStartCaseExperiment } = require(helperPath);
let receivedDependencies;
const result = await runStartCaseExperiment(
{ scenario: "Condition A test scenario" },
null,
{
condition: "A",
startCase: async (_input, dependencies) => {
receivedDependencies = dependencies;
return {
success: true,
situationGraph: { nodes: [], edges: [] },
assessment: null,
selectedQuestion: null,
summary: "test",
diagnostics: { validationStatus: "valid" },
};
},
},
);
expect(result.success).toBe(true);
});
it("--condition B with mock produces valid structured output", async () => {
const result = await runHelper(["--condition", "B", "Mock scenario"], {});
expect(result.exitCode).toBe(0);
const parsed = parseOutput(result);
expect(parsed.success).toBe(true);
});
it("Condition B schema JSON has importantUnknowns WITHOUT question field", async () => {
const { conditionBJsonSchema, conditionAJsonSchema } = require(
join(rootDir, "lib", "reconstruction", "schema-variant.js"),
);
const bIuProps = conditionBJsonSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items?.properties;
const aIuProps = conditionAJsonSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items?.properties;
expect(bIuProps).toHaveProperty("id");
expect(bIuProps).toHaveProperty("description");
expect(bIuProps).toHaveProperty("confidence");
expect(bIuProps).not.toHaveProperty("question");
// Verify Condition A (production) still HAS question
expect(aIuProps).toHaveProperty("id");
expect(aIuProps).toHaveProperty("description");
expect(aIuProps).toHaveProperty("confidence");
expect(aIuProps).toHaveProperty("question");
});
it("Condition B prompt file exists and contains no question-generation language", async () => {
const fs = require("fs");
const altPromptPath = join(rootDir, "prompts", "reconstruct-v0.5-b-noreq.md");
expect(fs.existsSync(altPromptPath)).toBe(true);
const content = fs.readFileSync(altPromptPath, "utf8");
// The B prompt should NOT instruct model to generate questions for importantUnknowns
expect(content).not.toMatch(/express that same uncertainty in \`question\`/i);
expect(content).not.toMatch(/natural direct question/i);
// But Condition A prompt does contain this language
const prodPromptPath = join(rootDir, "prompts", "reconstruct-v0.5.md");
const prodContent = fs.readFileSync(prodPromptPath, "utf8");
expect(prodContent).toMatch(/express that same uncertainty in \`question\`/i);
});
it("Condition B prompt has identical line count to Condition A prompt", async () => {
const fs = require("fs");
const altLines = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-b-noreq.md"), "utf8").split("\n").length;
const prodLines = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5.md"), "utf8").split("\n").length;
expect(altLines).toBe(prodLines);
});
it("Condition B prompt differs from Condition A only in Rule 7a and output format example", async () => {
const fs = require("fs");
const altContent = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-b-noreq.md"), "utf8");
const prodContent = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5.md"), "utf8");
expect(altContent).not.toBe(prodContent);
expect(altContent).toMatch(/express its canonical uncertainty meaning as a description in \`description\` with strict semantic fidelity/i);
});
it("Condition B schema validates importantUnknowns WITHOUT question field required", async () => {
const { conditionBJsonSchema } = require(
join(rootDir, "lib", "reconstruction", "schema-variant.js"),
);
const bIuItems = conditionBJsonSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items;
expect(bIuItems).toBeDefined();
if (bIuItems && bIuItems.required) {
expect(bIuItems.required).not.toContain("question");
}
});
});
});