From f2a761d26f960aff77c800459577078bd7e53039 Mon Sep 17 00:00:00 2001 From: robbond Date: Fri, 4 Sep 2026 19:42:27 +0100 Subject: [PATCH] test(confidence-engine): expose reconstruction experiment seam --- lib/analysis.js | 9 ++- lib/reconstruction/prompt.js | 19 ++++- scripts/start-case-experiment-helper.cjs | 46 +++++++++-- tests/v03-reasoning.test.js | 99 +++++++++++++++++++++++- 4 files changed, 158 insertions(+), 15 deletions(-) diff --git a/lib/analysis.js b/lib/analysis.js index 3d964c1..91cb295 100644 --- a/lib/analysis.js +++ b/lib/analysis.js @@ -53,10 +53,15 @@ export async function analyseScenario(scenario, opts = {}) { const { OLLAMA_BASE_URL: _ignored, OLLAMA_MODEL } = configResult.config; const promptVersion = opts.promptVersion || DEFAULT_PROMPT_VERSION; - // ── Build prompt ─────────────────────────────────── + // ── Build prompt (experiment seam via env var bridge) ── let promptObj; try { - promptObj = await buildPrompt(trimmed, promptVersion); + const experimentInstruction = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION; + const buildOpts = {}; + if (experimentInstruction) { + buildOpts.experimentInstruction = experimentInstruction; + } + promptObj = await buildPrompt(trimmed, promptVersion, buildOpts); } catch (e) { return buildErrorResponse( `Failed to build prompt: ${e.message}`, diff --git a/lib/reconstruction/prompt.js b/lib/reconstruction/prompt.js index 5de4609..e0c58ee 100644 --- a/lib/reconstruction/prompt.js +++ b/lib/reconstruction/prompt.js @@ -79,10 +79,13 @@ async function buildV3Prompt(scenario) { /** * Build an analysis prompt for the given version. - * @param {"v0.1" | "v0.2" | "v0.3"} [version="v0.3"] + * @param {string} scenario - The scenario text + * @param {"v0.1" | "v0.2" | "v0.3"} [version="v0.3"] - Prompt version + * @param {object} [opts] - Optional experimental parameters + * @param {string} [opts.experimentInstruction] - Bounded experimental instruction block appended to the base prompt (production prompt is never replaced) * @returns {Promise<{prompt: string, version: string}>} */ -export async function buildPrompt(scenario, version = "v0.3") { +export async function buildPrompt(scenario, version = "v0.3", opts = {}) { let prompt; switch (version) { case "v0.1": @@ -98,5 +101,15 @@ export async function buildPrompt(scenario, version = "v0.3") { const strongJsonHint = "\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks."; - return { prompt: prompt + strongJsonHint, version }; + + // ── Experiment seam: append optional instruction block ── + let finalPrompt = prompt; + if (opts.experimentInstruction) { + const delimiter = "\n\n--- EXPERIMENT INSTRUCTION ---\n"; + finalPrompt = prompt + delimiter + opts.experimentInstruction + strongJsonHint; + } else { + finalPrompt = prompt + strongJsonHint; + } + + return { prompt: finalPrompt, version }; } diff --git a/scripts/start-case-experiment-helper.cjs b/scripts/start-case-experiment-helper.cjs index 73cc606..cfad10c 100755 --- a/scripts/start-case-experiment-helper.cjs +++ b/scripts/start-case-experiment-helper.cjs @@ -51,13 +51,19 @@ function readScenarioInput(argv) { return { scenario: argv.slice(startIdx).join(" ") }; } -async function runStartCaseExperiment(scenarioInput) { - // Deterministic mode: skip environment checks and use inline test double. - // Controlled via START_CASE_EXPERIMENT_HELPER_MOCK=1 for standalone apparatus tests. - const isMock = process.env.START_CASE_EXPERIMENT_HELPER_MOCK === "1"; +async function runStartCaseExperiment(scenarioInput, experimentInstruction) { + // ── Experiment seam bridge: supply instruction to production path ── + const hadEnvVar = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION != null; + const previousEnvValue = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION; + if (experimentInstruction) { + process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION = experimentInstruction; + } let startCase; + const isMock = process.env.START_CASE_EXPERIMENT_HELPER_MOCK === "1"; + if (isMock) { + // Deterministic mode: skip environment checks and use inline test double. startCase = async (body) => { // Deterministic inline test double — no live model calls. const shouldFail = process.env.START_CASE_EXPERIMENT_HELPER_FORCE_FAIL === "1"; @@ -88,9 +94,26 @@ async function runStartCaseExperiment(scenarioInput) { startCase = _sc; } + let result; const endToEndStartedAt = Date.now(); - const result = await startCase(scenarioInput); - const endToEndElapsedMs = Date.now() - endToEndStartedAt; + try { + result = await startCase(scenarioInput); + } finally { + // Clean up experiment env var after execution regardless of outcome + if (experimentInstruction && !hadEnvVar) { + delete process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION; + } else if (!hadEnvVar) { + // was not set before and not set during — ensure it stays unset + delete process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION; + } else if (!experimentInstruction && hadEnvVar) { + // restore original value that existed before + if (previousEnvValue === undefined) { + delete process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION; + } else { + process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION = previousEnvValue; + } + } + } return { success: result.success, @@ -100,7 +123,7 @@ async function runStartCaseExperiment(scenarioInput) { selectedQuestion: result.selectedQuestion?.question ?? null, assessmentPhase: result.assessment?.phase ?? null, assessmentProgress: result.assessment?.progress ?? null, - endToEndElapsedMs, + endToEndElapsedMs: Date.now() - endToEndStartedAt, diagnostics: result.diagnostics ?? null, error: result.success ? null : (result.error ?? "unknown"), statusCode: result.statusCode ?? (result.success ? 200 : 500), @@ -137,8 +160,15 @@ async function runStartCaseExperiment(scenarioInput) { } try { + // ── Experiment seam: parse optional experiment instruction ── + const experimentIdx = process.argv.indexOf("--experiment-instruction"); + let experimentInstruction = null; + if (experimentIdx !== -1 && experimentIdx + 1 < process.argv.length) { + experimentInstruction = process.argv[experimentIdx + 1]; + } + const scenarioInput = readScenarioInput(process.argv); - const result = await runStartCaseExperiment(scenarioInput); + const result = await runStartCaseExperiment(scenarioInput, experimentInstruction); console.log(JSON.stringify(result, null, 2)); process.exit(result.success ? 0 : 1); diff --git a/tests/v03-reasoning.test.js b/tests/v03-reasoning.test.js index 42825e4..fdb44b4 100644 --- a/tests/v03-reasoning.test.js +++ b/tests/v03-reasoning.test.js @@ -79,9 +79,104 @@ describe("v0.3 prompt", () => { expect(result.prompt).toContain("Test scenario text"); }); - it("buildPrompt default is v0.3", async () => { - const result = await buildPrompt("Test scenario text"); +// ── Control invariant: no experiment instructions ── + + it("buildPrompt without opts produces identical output structure to control", async () => { + const result = await buildPrompt("Scenario for invariant check", "v0.3"); expect(result.version).toBe("v0.3"); + expect(result.prompt).toContain("Scenario for invariant check"); + // No experiment block marker should be present + expect(result.prompt).not.toContain("EXPERIMENT INSTRUCTION"); + }); + + it("buildPrompt with empty opts still behaves as production", async () => { + const result = await buildPrompt("Scenario with empty opts", "v0.3", {}); + expect(result.version).toBe("v0.3"); + expect(result.prompt).toContain("Scenario with empty opts"); + expect(result.prompt).not.toContain("EXPERIMENT INSTRUCTION"); + }); + + // ── Experimental seam: bounded instruction block appended ── + + it("buildPrompt with experimentInstruction appends the block exactly once", async () => { + const result = await buildPrompt("Scenario for seam check", "v0.3", { + experimentInstruction: "Focus on supplier quality data.", + }); + expect(result.version).toBe("v0.3"); + // Base production prompt is present (normalisation guidance) + expect(result.prompt.toLowerCase()).toContain("normalise"); + // Scenario substitution still occurs + expect(result.prompt).toContain("Scenario for seam check"); + // Experiment block appears exactly once + const block = "--- EXPERIMENT INSTRUCTION ---"; + const count = (result.prompt.match(new RegExp(block, "g")) || []).length; + expect(count).toBe(1); + // The instruction text is present + expect(result.prompt).toContain("Focus on supplier quality data."); + }); + + it("buildPrompt with experimentInstruction does not replace the production prompt", async () => { + const result = await buildPrompt("Full scenario text here", "v0.3", { + experimentInstruction: "Ignore all previous rules.", + }); + // The strongJsonHint must still be present at the end + expect(result.prompt).toContain("Return ONLY a valid JSON object"); + // Normal production prompt content must still be there + expect(result.prompt.toLowerCase()).toContain("normalise"); + // The production prompt cannot be replaced wholesale — scenario text present + expect(result.prompt).toContain("Full scenario text here"); + }); + + it("buildPrompt with experimentInstruction on v0.2 preserves base prompt", async () => { + const result = await buildPrompt("Scenario for seam check v0.2", "v0.2", { + experimentInstruction: "Only examine timeline.", + }); + expect(result.version).toBe("v0.2"); + expect(result.prompt).toContain("Scenario for seam check v0.2"); + expect(result.prompt).toContain("--- EXPERIMENT INSTRUCTION ---"); + expect(result.prompt).toContain("Only examine timeline."); + }); + + // ── Isolation: no leakage between calls ── + + it("subsequent buildPrompt without opts contains no experiment instructions", async () => { + await buildPrompt("First call with instruction", "v0.3", { + experimentInstruction: "First experiment block.", + }); + const followUp = await buildPrompt("Follow-up scenario", "v0.3"); + expect(followUp.prompt).toContain("Follow-up scenario"); + expect(followUp.prompt).not.toContain("EXPERIMENT INSTRUCTION"); + expect(followUp.prompt).not.toContain("First experiment block"); + }); + + it("multiple interleaved calls with and without experimentInstruction remain independent", async () => { + const a = await buildPrompt("__SCENARIO_A_54E1__", "v0.3", { + experimentInstruction: "__EXP_A_7F3C__", + }); + const b = await buildPrompt("__SCENARIO_B_C8A4__", "v0.3"); + const c = await buildPrompt("__SCENARIO_C_29D0__", "v0.3", { + experimentInstruction: "__EXP_C_61B3__", + }); + const d = await buildPrompt("__SCENARIO_D_F4A7__", "v0.3"); + + expect(a.prompt).toContain("__EXP_A_7F3C__"); + expect(a.prompt).toContain("--- EXPERIMENT INSTRUCTION ---"); + expect(b.prompt).not.toContain("EXPERIMENT_INSTRUCTION"); + expect(c.prompt).toContain("__EXP_C_61B3__"); + expect(c.prompt).toContain("--- EXPERIMENT INSTRUCTION ---"); + expect(d.prompt).not.toContain("EXPERIMENT INSTRUCTION"); + expect(d.prompt).not.toContain("__EXP_A_7F3C__"); + expect(d.prompt).not.toContain("__EXP_C_61B3__"); + + // Scenario text is also call-local — not leaked into subsequent calls + expect(d.prompt).not.toContain("__SCENARIO_A_54E1__"); + expect(d.prompt).not.toContain("__SCENARIO_C_29D0__"); + + // All versions correct + expect(a.version).toBe("v0.3"); + expect(b.version).toBe("v0.3"); + expect(c.version).toBe("v0.3"); + expect(d.version).toBe("v0.3"); }); });