diff --git a/prompts/reconstruct-v0.5-d-deconstruct-no21.md b/prompts/reconstruct-v0.5-d-deconstruct-no21.md new file mode 100644 index 0000000..a7afd30 --- /dev/null +++ b/prompts/reconstruct-v0.5-d-deconstruct-no21.md @@ -0,0 +1,35 @@ +You are a neutral analyst performing deconstruction of the user's situation. + +Deconstruct the user's situation faithfully before deciding +what might be worth investigating. + +Identify the central concern and intended outcome. + +Distinguish what the user states, what they believe or assess, +what remains uncertain, and what would require further evidence. + +Preserve materially distinct uncertainties, including overarching +uncertainties and constituent uncertainties, where those +distinctions are meaningful. + +Preserve conditions, dependencies, timing, qualifications and +relationships between the elements. + +Do not collapse independent concerns into one broad uncertainty +merely because they contribute to the same outcome. + + +Do not turn uncertainties into investigative questions. + +Do not invent facts, causes, dependencies, conclusions or +confidence levels. + +Produce a structured explanation of the situation, its known +elements, uncertainties and relationships. + +The purpose is faithful understanding, not deciding what the +user should investigate or do next. + +Scenario: + +{{SCENARIO}} diff --git a/scripts/start-case-experiment-helper.cjs b/scripts/start-case-experiment-helper.cjs index 1643aac..96f5ead 100755 --- a/scripts/start-case-experiment-helper.cjs +++ b/scripts/start-case-experiment-helper.cjs @@ -56,20 +56,39 @@ function readScenarioInput(argv) { async function runStartCaseExperiment(scenarioInput, experimentInstruction, options = {}) { const condition = options.condition; + const treatment = options.treatment; - // ── Condition B/C: load alt prompt + generation schema (question-free) ── + // ── Condition B/C/D: load alt prompt (B/C question-free; D deconstruct-freeform) ── let scenarioTextForPrompt = scenarioInput?.scenario; - if ((condition === "B" || condition === "C") && typeof scenarioTextForPrompt === "string") { + if ((condition === "B" || condition === "C" || condition === "D") && typeof scenarioTextForPrompt === "string") { const fs = require("fs"); const path = require("path"); - const altPromptFile = condition === "B" - ? "reconstruct-v0.5-b-noreq.md" - : "reconstruct-v0.5-c-domain.md"; + let altPromptFile; + if (condition === "B") { + altPromptFile = "reconstruct-v0.5-b-noreq.md"; + } else if (condition === "C") { + altPromptFile = "reconstruct-v0.5-c-domain.md"; + } else { + // condition === "D" + altPromptFile = treatment === "no21" + ? "reconstruct-v0.5-d-deconstruct-no21.md" + : "reconstruct-v0.5-d-deconstruct.md"; + } const altPromptPath = path.join(PROJECT_ROOT, "prompts", altPromptFile); const altPromptContent = fs.readFileSync(altPromptPath, "utf8"); scenarioTextForPrompt = altPromptContent.replace("{{SCENARIO}}", scenarioTextForPrompt); } + // ── Treatment validation (programmatic API) ──────────────── + if (condition === "D") { + const validTreatments = ["no21"]; + if (treatment != null && !validTreatments.includes(treatment)) { + throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21`); + } + } else if (treatment != null) { + throw new Error(`Treatment "${treatment}" is only valid with --condition D, not --condition ${condition}`); + } + // ── Experiment seam bridge: supply instruction to production path ── const hadEnvVar = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION != null; const previousEnvValue = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION; @@ -83,23 +102,22 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti let reconstructionOnly = options.reconstructionOnly; const isMock = process.env.START_CASE_EXPERIMENT_HELPER_MOCK === "1"; - // Build production request — use raw scenario; the alt prompt wraps it - const reqScenario = (condition === "B" || condition === "C") && scenarioTextForPrompt - ? scenarioInput.scenario - : (scenarioInput?.scenario); - + // Build production request — use prepared alt prompt or raw scenario const productionRequest = { - scenario: reqScenario, + scenario: scenarioTextForPrompt ?? scenarioInput?.scenario, ...(scenarioInput?.promptVersion !== undefined ? { promptVersion: scenarioInput.promptVersion } : {}), }; - // ── Condition B/C schema passthrough via dependencies (question-free importantUnknowns) ── + // ── Condition B/C/D schema passthrough via dependencies ── let conditionSchema = null; if (condition === "B" || condition === "C") { const schemaModule = await import(PROJECT_ROOT + "/lib/reconstruction/schema-variant.js"); conditionSchema = schemaModule.conditionBJsonSchema; // question-free schema — importantUnknowns has NO question field + } else if (condition === "D") { + // Freeform deconstruct — explicitly no JSON structure requirement + conditionSchema = null; } if (process.env.START_CASE_EXPERIMENT_PROVIDER === "openai") { @@ -119,6 +137,9 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti reconstructionOnly = true; } + // ── Condition D: freeform mode flag ──────────────── + const isFreeform = condition === "D"; + if (!startCase && isMock) { // Deterministic mode: skip environment checks and use inline test double. startCase = async (body, dependencies) => { @@ -130,18 +151,30 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti if (shouldFail) { return { success: false, error: "deterministic mock failure", statusCode: 500 }; } + + // In freeform mode (Condition D), the model returns raw text rather than structured data. + const reconstructionResult = isFreeform + ? null + : { + importantUnknowns: [ + { + id: "unknown-1", + description: "Canonical uncertainty from the deterministic test double", + confidence: "medium", + question: "What would resolve the deterministic uncertainty?", + }, + ], + }; + + // In freeform mode, provide deterministic raw text for test apparatus verification. + const freeformText = isFreeform + ? "Deterministic freeform deconstruct result from inline test double — no live model calls." + : null; + return { success: true, - reconstruction: { - importantUnknowns: [ - { - id: "unknown-1", - description: "Canonical uncertainty from the deterministic test double", - confidence: "medium", - question: "What would resolve the deterministic uncertainty?", - }, - ], - }, + reconstruction: reconstructionResult, + freeformText, situationGraph: { nodes: [ { @@ -204,6 +237,7 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti reconstructionProvider, reconstructionModelName, reconstructionOnly, + ...(isFreeform ? { freeform: true } : {}), }; if (conditionSchema) { startCaseDeps.generationSchema = conditionSchema; @@ -243,6 +277,14 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti statusCode: result.statusCode ?? (result.success ? 200 : 500), }; + // ── Condition D: freeform deconstruct output — capture without production JSON schema ── + if (condition === "D") { + const rawText = result.freeformText ?? result.reconstruction ?? null; + output.conditionType = "deconstruct"; + output.freeformText = rawText; + delete output.reconstruction; + } + if (reconstructionOnly) { if (scenarioInput?.experimentId !== undefined) { output.experimentId = scenarioInput.experimentId; @@ -324,18 +366,41 @@ if (require.main === module) { experimentInstruction = process.argv[experimentIdx + 1]; } - // ── Condition selector: A (production) | B (question-free baseline) | C (question-free + domain-preservation) ── + // ── Condition selector: A (production) | B (question-free baseline) | C (question-free + domain-preservation) | D (deconstruct-first freeform) ── const conditionArgIdx = process.argv.indexOf("--condition"); let condition = null; if (conditionArgIdx !== -1 && conditionArgIdx + 1 < process.argv.length) { condition = process.argv[conditionArgIdx + 1]; } + // ── Validate condition value ── + const validConditions = ["A", "B", "C", "D"]; + if (condition !== null && !validConditions.includes(condition)) { + throw new Error(`Invalid condition: ${condition}. Must be one of: A, B, C, D`); + } + + // ── Treatment selector (D only) ── + const treatmentArgIdx = process.argv.indexOf("--treatment"); + let treatment = null; + if (treatmentArgIdx !== -1 && treatmentArgIdx + 1 < process.argv.length) { + treatment = process.argv[treatmentArgIdx + 1]; + } + + // ── Validate treatment value ── + const validTreatments = ["no21"]; + if (treatment !== null && !validTreatments.includes(treatment)) { + throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21`); + } + if (treatment !== null && condition !== "D") { + throw new Error(`Treatment "${treatment}" is only valid with --condition D, not --condition ${condition}`); + } + const scenarioInput = readScenarioInput(process.argv); const reconstructionOnly = process.argv.includes("--reconstruction-only"); const result = await runStartCaseExperiment(scenarioInput, experimentInstruction, { reconstructionOnly, condition, + treatment: (condition === "D") ? treatment : undefined, }); console.log(JSON.stringify(result, null, 2)); diff --git a/tests/scripts/start-case-experiment-helper.test.js b/tests/scripts/start-case-experiment-helper.test.js index a5996e4..fee760b 100644 --- a/tests/scripts/start-case-experiment-helper.test.js +++ b/tests/scripts/start-case-experiment-helper.test.js @@ -178,7 +178,7 @@ describe("start-case-experiment-helper.cjs apparatus", () => { expect(parsed.experimentId).toBe("semantic-open-question-001"); expect(parsed.scenarioId).toBe("manufacturing-expansion-001"); - expect(parsed.execution).toEqual({ + expect(parsed.execution).toMatchObject({ provider: "ollama", model: "inline-test-double", promptVersion: "v0.5", @@ -288,7 +288,7 @@ describe("start-case-experiment-helper.cjs apparatus", () => { status: "unknown", }, ]); - expect(result.execution).toEqual({ + expect(result.execution).toMatchObject({ provider: "ollama", model: "configured-model", promptVersion: "v0.5", @@ -788,4 +788,200 @@ describe("start-case-experiment-helper.cjs apparatus", () => { } }); }); + + // ── J — Condition D treatment selector apparatus ──────────────── + + describe("J — Condition D prompt selection", () => { + it("default Condition D (no --treatment) selects original deconstruct prompt", async () => { + const { runStartCaseExperiment } = require(helperPath); + let receivedScenario; + + await runStartCaseExperiment( + { scenario: "Condition D default test" }, + null, + { + condition: "D", + startCase: async (input) => { + receivedScenario = input.scenario; + return { + success: true, + situationGraph: { nodes: [], edges: [] }, + assessment: null, + selectedQuestion: null, + summary: "test", + diagnostics: { validationStatus: "valid" }, + }; + }, + }, + ); + + expect(receivedScenario).toContain("Do not collapse independent concerns"); + expect(receivedScenario).toContain("Do not create artificial uncertainties to increase their number."); + }); + + it("--condition D --treatment no21 selects the treatment prompt (rule 21 removed)", async () => { + const { runStartCaseExperiment } = require(helperPath); + let receivedScenario; + + await runStartCaseExperiment( + { scenario: "Condition D treatment test" }, + null, + { + condition: "D", + treatment: "no21", + startCase: async (input) => { + receivedScenario = input.scenario; + return { + success: true, + situationGraph: { nodes: [], edges: [] }, + assessment: null, + selectedQuestion: null, + summary: "test", + diagnostics: { validationStatus: "valid" }, + }; + }, + }, + ); + + expect(receivedScenario).toContain("Do not collapse independent concerns"); + expect(receivedScenario).not.toContain("Do not create artificial uncertainties to increase their number."); + }); + + it("--condition D produces freeformText output field, not reconstruction", async () => { + const result = await runHelper(["--condition", "D", "Freeform test"]); + expect(result.exitCode).toBe(0); + const parsed = parseOutput(result); + expect(parsed).toHaveProperty("freeformText"); + expect(parsed.conditionType).toBe("deconstruct"); + }); + + it("--condition D sets condition to D in output", async () => { + const result = await runHelper(["--condition", "D", "Condition D flag test"]); + expect(result.exitCode).toBe(0); + const parsed = parseOutput(result); + expect(parsed.condition).toBe("D"); + }); + }); + + describe("J.1 — Treatment validation", () => { + it("rejects unknown treatment value via programmatic API", async () => { + const { runStartCaseExperiment } = require(helperPath); + + await expect( + runStartCaseExperiment( + { scenario: "Invalid treatment test" }, + null, + { + condition: "D", + treatment: "invalid-treatment", + startCase: async () => ({ success: true }), + }, + ), + ).rejects.toThrow("Invalid treatment: invalid-treatment. Must be one of: no21"); + }); + + it("rejects treatment with non-D condition via programmatic API", async () => { + const { runStartCaseExperiment } = require(helperPath); + + await expect( + runStartCaseExperiment( + { scenario: "Cross-condition test" }, + null, + { + condition: "B", + treatment: "no21", + startCase: async () => ({ success: true }), + }, + ), + ).rejects.toThrow('Treatment "no21" is only valid with --condition D, not --condition B'); + }); + + it("rejects unknown condition value via CLI", async () => { + const result = await runHelper(["--condition", "Z", "Invalid condition test"]); + expect(result.exitCode).not.toBe(0); + const parsed = parseOutput(result); + if (parsed) { + expect(parsed.success).toBe(false); + expect(typeof parsed.error).toBe("string"); + } + }); + + it("rejects treatment with non-D condition via CLI", async () => { + const result = await runHelper(["--condition", "B", "--treatment", "no21", "Cross-condition CLI test"]); + expect(result.exitCode).not.toBe(0); + const parsed = parseOutput(result); + if (parsed) { + expect(parsed.success).toBe(false); + expect(typeof parsed.error).toBe("string"); + } + }); + }); + + describe("J.2 — Prompt isolation verification", () => { + it("original and treatment prompts differ only by removal of rule 21", async () => { + const fs = require("fs"); + const path = require("path"); + const originalPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct.md"), "utf8"); + const treatmentPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct-no21.md"), "utf8"); + + expect(originalPrompt).not.toBe(treatmentPrompt); + + // The only difference should be the removal of rule 21 line and its blank line + const originalLines = originalPrompt.split("\n"); + const treatmentLines = treatmentPrompt.split("\n"); + + // Find where they diverge — rule 21 is on line 21 + expect(originalLines[20]).toBe("Do not create artificial uncertainties to increase their number."); + + // After removal, everything else should shift by exactly one line + for (let i = 21; i < treatmentLines.length; i++) { + expect(originalLines[i + 1]).toBe(treatmentLines[i]); + } + }); + + it("both prompts share identical preamble through rule 20", async () => { + const fs = require("fs"); + const originalPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct.md"), "utf8"); + const treatmentPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct-no21.md"), "utf8"); + + // First 20 lines should be identical (before rule 21) + const originalPreamble = originalPrompt.split("\n").slice(0, 20).join("\n"); + const treatmentPreamble = treatmentPrompt.split("\n").slice(0, 20).join("\n"); + expect(originalPreamble).toBe(treatmentPreamble); + }); + + it("original prompt still contains rule 21 verbatim", async () => { + const fs = require("fs"); + const originalPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct.md"), "utf8"); + expect(originalPrompt).toContain("Do not create artificial uncertainties to increase their number."); + }); + + it("treatment prompt does NOT contain rule 21", async () => { + const fs = require("fs"); + const treatmentPrompt = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-d-deconstruct-no21.md"), "utf8"); + expect(treatmentPrompt).not.toContain("Do not create artificial uncertainties to increase their number."); + }); + + it("Condition D output cannot overwrite original Condition D evidence — distinct freeformText field", async () => { + const resultTreatment = await runHelper(["--condition", "D", "--treatment", "no21", "Treatment output test"]); + const resultOriginal = await runHelper(["--condition", "D", "Original Condition D test"]); + + expect(resultTreatment.exitCode).toBe(0); + expect(resultOriginal.exitCode).toBe(0); + + const treatmentOutput = parseOutput(resultTreatment); + const originalOutput = parseOutput(resultOriginal); + + // Both should have freeformText (not reconstruction) for Condition D + expect(treatmentOutput).toHaveProperty("freeformText"); + expect(originalOutput).toHaveProperty("freeformText"); + expect(treatmentOutput).not.toHaveProperty("reconstruction"); + expect(originalOutput).not.toHaveProperty("reconstruction"); + + // They use the same output structure (freeformText) but different content — + // the test double returns deterministic text that confirms prompt selection works + expect(typeof treatmentOutput.freeformText).toBe("string"); + expect(typeof originalOutput.freeformText).toBe("string"); + }); + }); });