feat(experiments): integrate Condition D/E apparatus
This commit is contained in:
@@ -0,0 +1,36 @@
|
|||||||
|
You are a neutral analyst performing deconstruction of the user's situation.
|
||||||
|
|
||||||
|
Deconstruct the user's situation faithfully before deciding
|
||||||
|
what might be worth investigating.
|
||||||
|
|
||||||
|
Identify the central concern and intended outcome.
|
||||||
|
|
||||||
|
Distinguish what the user states, what they believe or assess,
|
||||||
|
what remains uncertain, and what would require further evidence.
|
||||||
|
|
||||||
|
Preserve materially distinct uncertainties, including overarching
|
||||||
|
uncertainties and constituent uncertainties, where those
|
||||||
|
distinctions are meaningful.
|
||||||
|
|
||||||
|
Preserve conditions, dependencies, timing, qualifications and
|
||||||
|
relationships between the elements.
|
||||||
|
|
||||||
|
Do not collapse independent concerns into one broad uncertainty
|
||||||
|
merely because they contribute to the same outcome.
|
||||||
|
|
||||||
|
Do not create artificial uncertainties to increase their number.
|
||||||
|
|
||||||
|
Do not turn uncertainties into investigative questions.
|
||||||
|
|
||||||
|
Do not invent facts, causes, dependencies, conclusions or
|
||||||
|
confidence levels.
|
||||||
|
|
||||||
|
Produce a structured explanation of the situation, its known
|
||||||
|
elements, uncertainties and relationships.
|
||||||
|
|
||||||
|
The purpose is faithful understanding, not deciding what the
|
||||||
|
user should investigate or do next.
|
||||||
|
|
||||||
|
Scenario:
|
||||||
|
|
||||||
|
{{SCENARIO}}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
You are a neutral analyst performing deconstruction of the user's situation.
|
||||||
|
|
||||||
|
Deconstruct the user's situation faithfully before deciding
|
||||||
|
what might be worth investigating.
|
||||||
|
|
||||||
|
Identify the central concern and intended outcome.
|
||||||
|
|
||||||
|
Distinguish what the user states, what they believe or assess,
|
||||||
|
what remains uncertain, and what would require further evidence.
|
||||||
|
|
||||||
|
**Before considering which uncertainties matter most or how they combine, establish whether each can be investigated independently — without resolving the others.**
|
||||||
|
|
||||||
|
Preserve materially distinct uncertainties, including overarching
|
||||||
|
uncertainties and constituent uncertainties, where those
|
||||||
|
distinctions are meaningful.
|
||||||
|
|
||||||
|
Preserve conditions, dependencies, timing, qualifications and
|
||||||
|
relationships between the elements.
|
||||||
|
|
||||||
|
Do not collapse independent concerns into one broad uncertainty
|
||||||
|
merely because they contribute to the same outcome.
|
||||||
|
|
||||||
|
Do not create artificial uncertainties to increase their number.
|
||||||
|
|
||||||
|
Do not turn uncertainties into investigative questions.
|
||||||
|
|
||||||
|
Do not invent facts, causes, dependencies, conclusions or
|
||||||
|
confidence levels.
|
||||||
|
|
||||||
|
Produce a structured explanation of the situation, its known
|
||||||
|
elements, uncertainties and relationships.
|
||||||
|
|
||||||
|
The purpose is faithful understanding, not deciding what the
|
||||||
|
user should investigate or do next.
|
||||||
|
|
||||||
|
Scenario:
|
||||||
|
|
||||||
|
{{SCENARIO}}
|
||||||
@@ -58,9 +58,9 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
|
|||||||
const condition = options.condition;
|
const condition = options.condition;
|
||||||
const treatment = options.treatment;
|
const treatment = options.treatment;
|
||||||
|
|
||||||
// ── Condition B/C/D: load alt prompt (B/C question-free; D deconstruct-freeform) ──
|
// ── Condition B/C/D/E: load alt prompt (B/C question-free; D deconstruct-freeform; E investigability-first freeform) ──
|
||||||
let scenarioTextForPrompt = scenarioInput?.scenario;
|
let scenarioTextForPrompt = scenarioInput?.scenario;
|
||||||
if ((condition === "B" || condition === "C" || condition === "D") && typeof scenarioTextForPrompt === "string") {
|
if ((condition === "B" || condition === "C" || condition === "D" || condition === "E") && typeof scenarioTextForPrompt === "string") {
|
||||||
const fs = require("fs");
|
const fs = require("fs");
|
||||||
const path = require("path");
|
const path = require("path");
|
||||||
let altPromptFile;
|
let altPromptFile;
|
||||||
@@ -68,11 +68,13 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
|
|||||||
altPromptFile = "reconstruct-v0.5-b-noreq.md";
|
altPromptFile = "reconstruct-v0.5-b-noreq.md";
|
||||||
} else if (condition === "C") {
|
} else if (condition === "C") {
|
||||||
altPromptFile = "reconstruct-v0.5-c-domain.md";
|
altPromptFile = "reconstruct-v0.5-c-domain.md";
|
||||||
} else {
|
} else if (condition === "D") {
|
||||||
// condition === "D"
|
|
||||||
altPromptFile = treatment === "no21"
|
altPromptFile = treatment === "no21"
|
||||||
? "reconstruct-v0.5-d-deconstruct-no21.md"
|
? "reconstruct-v0.5-d-deconstruct-no21.md"
|
||||||
: "reconstruct-v0.5-d-deconstruct.md";
|
: "reconstruct-v0.5-d-deconstruct.md";
|
||||||
|
} else {
|
||||||
|
// condition === "E"
|
||||||
|
altPromptFile = "reconstruct-v0.5-e-investigability-first.md";
|
||||||
}
|
}
|
||||||
const altPromptPath = path.join(PROJECT_ROOT, "prompts", altPromptFile);
|
const altPromptPath = path.join(PROJECT_ROOT, "prompts", altPromptFile);
|
||||||
const altPromptContent = fs.readFileSync(altPromptPath, "utf8");
|
const altPromptContent = fs.readFileSync(altPromptPath, "utf8");
|
||||||
@@ -85,8 +87,13 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
|
|||||||
if (treatment != null && !validTreatments.includes(treatment)) {
|
if (treatment != null && !validTreatments.includes(treatment)) {
|
||||||
throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21`);
|
throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21`);
|
||||||
}
|
}
|
||||||
|
} else if (condition === "E") {
|
||||||
|
const validTreatments = ["e-investigability-first"];
|
||||||
|
if (treatment != null && !validTreatments.includes(treatment)) {
|
||||||
|
throw new Error(`Invalid treatment: ${treatment}. Must be one of: e-investigability-first`);
|
||||||
|
}
|
||||||
} else if (treatment != null) {
|
} else if (treatment != null) {
|
||||||
throw new Error(`Treatment "${treatment}" is only valid with --condition D, not --condition ${condition}`);
|
throw new Error(`Treatment "${treatment}" is only valid with --condition D or E, not --condition ${condition}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Experiment seam bridge: supply instruction to production path ──
|
// ── Experiment seam bridge: supply instruction to production path ──
|
||||||
@@ -110,13 +117,13 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
|
|||||||
: {}),
|
: {}),
|
||||||
};
|
};
|
||||||
|
|
||||||
// ── Condition B/C/D schema passthrough via dependencies ──
|
// ── Condition B/C/D/E schema passthrough via dependencies ──
|
||||||
let conditionSchema = null;
|
let conditionSchema = null;
|
||||||
if (condition === "B" || condition === "C") {
|
if (condition === "B" || condition === "C") {
|
||||||
const schemaModule = await import(PROJECT_ROOT + "/lib/reconstruction/schema-variant.js");
|
const schemaModule = await import(PROJECT_ROOT + "/lib/reconstruction/schema-variant.js");
|
||||||
conditionSchema = schemaModule.conditionBJsonSchema; // question-free schema — importantUnknowns has NO question field
|
conditionSchema = schemaModule.conditionBJsonSchema; // question-free schema — importantUnknowns has NO question field
|
||||||
} else if (condition === "D") {
|
} else if (condition === "D" || condition === "E") {
|
||||||
// Freeform deconstruct — explicitly no JSON structure requirement
|
// Freeform deconstruct / investigability-first — explicitly no JSON structure requirement
|
||||||
conditionSchema = null;
|
conditionSchema = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -137,8 +144,8 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
|
|||||||
reconstructionOnly = true;
|
reconstructionOnly = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Condition D: freeform mode flag ────────────────
|
// ── Condition D/E: freeform mode flag ────────────────
|
||||||
const isFreeform = condition === "D";
|
const isFreeform = condition === "D" || condition === "E";
|
||||||
|
|
||||||
if (!startCase && isMock) {
|
if (!startCase && isMock) {
|
||||||
// Deterministic mode: skip environment checks and use inline test double.
|
// Deterministic mode: skip environment checks and use inline test double.
|
||||||
@@ -277,10 +284,10 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti
|
|||||||
statusCode: result.statusCode ?? (result.success ? 200 : 500),
|
statusCode: result.statusCode ?? (result.success ? 200 : 500),
|
||||||
};
|
};
|
||||||
|
|
||||||
// ── Condition D: freeform deconstruct output — capture without production JSON schema ──
|
// ── Condition D/E: freeform deconstruct / investigability-first output — capture without production JSON schema ──
|
||||||
if (condition === "D") {
|
if (condition === "D" || condition === "E") {
|
||||||
const rawText = result.freeformText ?? result.reconstruction ?? null;
|
const rawText = result.freeformText ?? result.reconstruction ?? null;
|
||||||
output.conditionType = "deconstruct";
|
output.conditionType = condition === "D" ? "deconstruct" : "investigability-first";
|
||||||
output.freeformText = rawText;
|
output.freeformText = rawText;
|
||||||
delete output.reconstruction;
|
delete output.reconstruction;
|
||||||
}
|
}
|
||||||
@@ -374,12 +381,12 @@ if (require.main === module) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ── Validate condition value ──
|
// ── Validate condition value ──
|
||||||
const validConditions = ["A", "B", "C", "D"];
|
const validConditions = ["A", "B", "C", "D", "E"];
|
||||||
if (condition !== null && !validConditions.includes(condition)) {
|
if (condition !== null && !validConditions.includes(condition)) {
|
||||||
throw new Error(`Invalid condition: ${condition}. Must be one of: A, B, C, D`);
|
throw new Error(`Invalid condition: ${condition}. Must be one of: A, B, C, D, E`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Treatment selector (D only) ──
|
// ── Treatment selector (D and E only) ──
|
||||||
const treatmentArgIdx = process.argv.indexOf("--treatment");
|
const treatmentArgIdx = process.argv.indexOf("--treatment");
|
||||||
let treatment = null;
|
let treatment = null;
|
||||||
if (treatmentArgIdx !== -1 && treatmentArgIdx + 1 < process.argv.length) {
|
if (treatmentArgIdx !== -1 && treatmentArgIdx + 1 < process.argv.length) {
|
||||||
@@ -387,12 +394,16 @@ if (require.main === module) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ── Validate treatment value ──
|
// ── Validate treatment value ──
|
||||||
const validTreatments = ["no21"];
|
const validDTreatments = ["no21"];
|
||||||
if (treatment !== null && !validTreatments.includes(treatment)) {
|
const validETreatments = ["e-investigability-first"];
|
||||||
|
if (treatment !== null && condition === "D" && !validDTreatments.includes(treatment)) {
|
||||||
throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21`);
|
throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21`);
|
||||||
}
|
}
|
||||||
if (treatment !== null && condition !== "D") {
|
if (treatment !== null && condition === "E" && !validETreatments.includes(treatment)) {
|
||||||
throw new Error(`Treatment "${treatment}" is only valid with --condition D, not --condition ${condition}`);
|
throw new Error(`Invalid treatment: ${treatment}. Must be one of: e-investigability-first`);
|
||||||
|
}
|
||||||
|
if (treatment !== null && !["D", "E"].includes(condition)) {
|
||||||
|
throw new Error(`Treatment "${treatment}" is only valid with --condition D or E, not --condition ${condition}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const scenarioInput = readScenarioInput(process.argv);
|
const scenarioInput = readScenarioInput(process.argv);
|
||||||
@@ -400,7 +411,7 @@ if (require.main === module) {
|
|||||||
const result = await runStartCaseExperiment(scenarioInput, experimentInstruction, {
|
const result = await runStartCaseExperiment(scenarioInput, experimentInstruction, {
|
||||||
reconstructionOnly,
|
reconstructionOnly,
|
||||||
condition,
|
condition,
|
||||||
treatment: (condition === "D") ? treatment : undefined,
|
treatment: (condition === "D" || condition === "E") ? treatment : undefined,
|
||||||
});
|
});
|
||||||
|
|
||||||
console.log(JSON.stringify(result, null, 2));
|
console.log(JSON.stringify(result, null, 2));
|
||||||
|
|||||||
@@ -893,7 +893,7 @@ describe("start-case-experiment-helper.cjs apparatus", () => {
|
|||||||
startCase: async () => ({ success: true }),
|
startCase: async () => ({ success: true }),
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
).rejects.toThrow('Treatment "no21" is only valid with --condition D, not --condition B');
|
).rejects.toThrow('Treatment "no21" is only valid with --condition D or E, not --condition B');
|
||||||
});
|
});
|
||||||
|
|
||||||
it("rejects unknown condition value via CLI", async () => {
|
it("rejects unknown condition value via CLI", async () => {
|
||||||
@@ -984,6 +984,133 @@ describe("start-case-experiment-helper.cjs apparatus", () => {
|
|||||||
expect(typeof originalOutput.freeformText).toBe("string");
|
expect(typeof originalOutput.freeformText).toBe("string");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// ── K — Condition E investigability-first apparatus ────────
|
||||||
|
|
||||||
|
describe("K — Condition E investigability-first apparatus", () => {
|
||||||
|
it("--condition E selects the investigability-first prompt rather than deconstruct", async () => {
|
||||||
|
const { runStartCaseExperiment } = require(helperPath);
|
||||||
|
let receivedScenario;
|
||||||
|
|
||||||
|
await runStartCaseExperiment(
|
||||||
|
{ scenario: "Condition E test" },
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
condition: "E",
|
||||||
|
startCase: async (input) => {
|
||||||
|
receivedScenario = input.scenario;
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
situationGraph: { nodes: [], edges: [] },
|
||||||
|
assessment: null,
|
||||||
|
selectedQuestion: null,
|
||||||
|
summary: "test",
|
||||||
|
diagnostics: { validationStatus: "valid" },
|
||||||
|
};
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(receivedScenario).toContain("You are a neutral analyst performing deconstruction of the user's situation");
|
||||||
|
expect(receivedScenario).toContain("Do not create artificial uncertainties to increase their number.");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("--condition E --treatment e-investigability-first is accepted", async () => {
|
||||||
|
const { runStartCaseExperiment } = require(helperPath);
|
||||||
|
|
||||||
|
await runStartCaseExperiment(
|
||||||
|
{ scenario: "Condition E treatment test" },
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
condition: "E",
|
||||||
|
treatment: "e-investigability-first",
|
||||||
|
startCase: async (input) => {
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
situationGraph: { nodes: [], edges: [] },
|
||||||
|
assessment: null,
|
||||||
|
selectedQuestion: null,
|
||||||
|
summary: "test",
|
||||||
|
diagnostics: { validationStatus: "valid" },
|
||||||
|
};
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
// If we reach here without an error, the treatment was accepted
|
||||||
|
});
|
||||||
|
|
||||||
|
it("--condition E produces freeformText output field via CLI mock", async () => {
|
||||||
|
const result = await runHelper(["--condition", "E", "Freeform test"]);
|
||||||
|
expect(result.exitCode).toBe(0);
|
||||||
|
const parsed = parseOutput(result);
|
||||||
|
expect(parsed).toHaveProperty("freeformText");
|
||||||
|
expect(typeof parsed.freeformText).toBe("string");
|
||||||
|
expect(parsed.conditionType).toBe("investigability-first");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("--condition E sets condition to E in output", async () => {
|
||||||
|
const result = await runHelper(["--condition", "E", "Condition E flag test"]);
|
||||||
|
expect(result.exitCode).toBe(0);
|
||||||
|
const parsed = parseOutput(result);
|
||||||
|
expect(parsed.condition).toBe("E");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("--condition E does not return reconstruction field in output", async () => {
|
||||||
|
const result = await runHelper(["--condition", "E", "No reconstruction test"]);
|
||||||
|
expect(result.exitCode).toBe(0);
|
||||||
|
const parsed = parseOutput(result);
|
||||||
|
expect(parsed).not.toHaveProperty("reconstruction");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("--condition E --reconstruction-only sets execution.condition to E", async () => {
|
||||||
|
const result = await runHelper(["--condition", "E", "--reconstruction-only", "Reconstruction only test"]);
|
||||||
|
expect(result.exitCode).toBe(0);
|
||||||
|
const parsed = parseOutput(result);
|
||||||
|
expect(parsed.execution).toBeDefined();
|
||||||
|
expect(parsed.execution.condition).toBe("E");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("--condition E rejects unknown treatment via programmatic API", async () => {
|
||||||
|
const { runStartCaseExperiment } = require(helperPath);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
runStartCaseExperiment(
|
||||||
|
{ scenario: "Invalid treatment test" },
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
condition: "E",
|
||||||
|
treatment: "invalid-treatment",
|
||||||
|
startCase: async () => ({ success: true }),
|
||||||
|
},
|
||||||
|
),
|
||||||
|
).rejects.toThrow("Invalid treatment: invalid-treatment. Must be one of: e-investigability-first");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("--condition E rejects treatment with non-E condition via programmatic API", async () => {
|
||||||
|
const { runStartCaseExperiment } = require(helperPath);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
runStartCaseExperiment(
|
||||||
|
{ scenario: "Cross-condition test" },
|
||||||
|
null,
|
||||||
|
{
|
||||||
|
condition: "B",
|
||||||
|
treatment: "e-investigability-first",
|
||||||
|
startCase: async () => ({ success: true }),
|
||||||
|
},
|
||||||
|
),
|
||||||
|
).rejects.toThrow('Treatment "e-investigability-first" is only valid with --condition D or E, not --condition B');
|
||||||
|
});
|
||||||
|
|
||||||
|
it("--condition E exposes non-empty freeformText (mock)", async () => {
|
||||||
|
const result = await runHelper(["--condition", "E", "Freeform content test"]);
|
||||||
|
expect(result.exitCode).toBe(0);
|
||||||
|
const parsed = parseOutput(result);
|
||||||
|
expect(parsed.freeformText.length).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
});
|
});
|
||||||
|
|
||||||
// ── Evidence persistence wrapper tests ─────────────────────────────
|
// ── Evidence persistence wrapper tests ─────────────────────────────
|
||||||
|
|||||||
Reference in New Issue
Block a user