feat(confidence-engine): add CE-ULI-08 two-condition uncertainty comparison apparatus

Adds experimental apparatus for testing whether removing the question field from importantUnknowns in reconstruction output improves semantic fidelity. New files: - lib/reconstruction/schema-variant.js (conditionB schema + production passthrough) - prompts/reconstruct-v0.5-b-noreq.md (alt prompt, identical line count to prod) Changed files: - lib/graph/orchestrator.js (pass generationSchema through dependencies) - lib/analysis.js (accept opts.generationSchema fallback to original schema) - scripts/start-case-experiment-helper.cjs (--condition B/A flag support) - tests/scripts/start-case-experiment-helper.test.js (8 new deterministic tests) 43 targeted deterministic tests PASS. Production callers use original prompt/schema unchanged. No live model calls. No full suite evidence available. Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-10-12 07:33:31 +01:00
parent a54b36a2b2
commit 3b04d05355
7 changed files with 544 additions and 5 deletions
@@ -654,4 +654,138 @@ describe("start-case-experiment-helper.cjs apparatus", () => {
      expect(typeof parsed.failureReason).toBe("string");
    });
  });
+
+  // ── I — Condition B schema variant apparatus ────────────────────
+
+  describe("I — Condition B schema variant apparatus", () => {
+    it("--condition B passes generationSchema through the startCase seam", async () => {
+      const { runStartCaseExperiment } = require(helperPath);
+      let receivedDependencies;
+
+      const result = await runStartCaseExperiment(
+        { scenario: "Condition B test scenario" },
+        null,
+        {
+          condition: "B",
+          startCase: async (_input, dependencies) => {
+            receivedDependencies = dependencies;
+            return {
+              success: true,
+              situationGraph: { nodes: [], edges: [] },
+              assessment: null,
+              selectedQuestion: null,
+              summary: "test",
+              diagnostics: { validationStatus: "valid" },
+            };
+          },
+        },
+      );
+
+      expect(result.success).toBe(true);
+      expect(receivedDependencies.generationSchema).toBeDefined();
+      // Condition B schema: importantUnknowns items should NOT have question property
+      const iuSchema = receivedDependencies.generationSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items;
+      if (iuSchema && iuSchema.properties) {
+        expect(iuSchema.properties).not.toHaveProperty("question");
+      }
+    });
+
+    it("--condition A uses production schema with question required", async () => {
+      const { runStartCaseExperiment } = require(helperPath);
+      let receivedDependencies;
+
+      const result = await runStartCaseExperiment(
+        { scenario: "Condition A test scenario" },
+        null,
+        {
+          condition: "A",
+          startCase: async (_input, dependencies) => {
+            receivedDependencies = dependencies;
+            return {
+              success: true,
+              situationGraph: { nodes: [], edges: [] },
+              assessment: null,
+              selectedQuestion: null,
+              summary: "test",
+              diagnostics: { validationStatus: "valid" },
+            };
+          },
+        },
+      );
+
+      expect(result.success).toBe(true);
+    });
+
+    it("--condition B with mock produces valid structured output", async () => {
+      const result = await runHelper(["--condition", "B", "Mock scenario"], {});
+      expect(result.exitCode).toBe(0);
+      const parsed = parseOutput(result);
+      expect(parsed.success).toBe(true);
+    });
+
+    it("Condition B schema JSON has importantUnknowns WITHOUT question field", async () => {
+      const { conditionBJsonSchema, conditionAJsonSchema } = require(
+        join(rootDir, "lib", "reconstruction", "schema-variant.js"),
+      );
+
+      const bIuProps = conditionBJsonSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items?.properties;
+      const aIuProps = conditionAJsonSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items?.properties;
+
+      expect(bIuProps).toHaveProperty("id");
+      expect(bIuProps).toHaveProperty("description");
+      expect(bIuProps).toHaveProperty("confidence");
+      expect(bIuProps).not.toHaveProperty("question");
+
+      // Verify Condition A (production) still HAS question
+      expect(aIuProps).toHaveProperty("id");
+      expect(aIuProps).toHaveProperty("description");
+      expect(aIuProps).toHaveProperty("confidence");
+      expect(aIuProps).toHaveProperty("question");
+    });
+
+    it("Condition B prompt file exists and contains no question-generation language", async () => {
+      const fs = require("fs");
+      const altPromptPath = join(rootDir, "prompts", "reconstruct-v0.5-b-noreq.md");
+
+      expect(fs.existsSync(altPromptPath)).toBe(true);
+      const content = fs.readFileSync(altPromptPath, "utf8");
+
+      // The B prompt should NOT instruct model to generate questions for importantUnknowns
+      expect(content).not.toMatch(/express that same uncertainty in \`question\`/i);
+      expect(content).not.toMatch(/natural direct question/i);
+
+      // But Condition A prompt does contain this language
+      const prodPromptPath = join(rootDir, "prompts", "reconstruct-v0.5.md");
+      const prodContent = fs.readFileSync(prodPromptPath, "utf8");
+      expect(prodContent).toMatch(/express that same uncertainty in \`question\`/i);
+    });
+
+    it("Condition B prompt has identical line count to Condition A prompt", async () => {
+      const fs = require("fs");
+      const altLines = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-b-noreq.md"), "utf8").split("\n").length;
+      const prodLines = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5.md"), "utf8").split("\n").length;
+      expect(altLines).toBe(prodLines);
+    });
+
+    it("Condition B prompt differs from Condition A only in Rule 7a and output format example", async () => {
+      const fs = require("fs");
+      const altContent = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-b-noreq.md"), "utf8");
+      const prodContent = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5.md"), "utf8");
+
+      expect(altContent).not.toBe(prodContent);
+      expect(altContent).toMatch(/express its canonical uncertainty meaning as a description in \`description\` with strict semantic fidelity/i);
+    });
+
+    it("Condition B schema validates importantUnknowns WITHOUT question field required", async () => {
+      const { conditionBJsonSchema } = require(
+        join(rootDir, "lib", "reconstruction", "schema-variant.js"),
+      );
+
+      const bIuItems = conditionBJsonSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items;
+      expect(bIuItems).toBeDefined();
+      if (bIuItems && bIuItems.required) {
+        expect(bIuItems.required).not.toContain("question");
+      }
+    });
+  });
 });