diff --git a/docs/experiment-current-understanding-apparatus.md b/docs/experiment-current-understanding-apparatus.md index 54e8439..9658255 100644 --- a/docs/experiment-current-understanding-apparatus.md +++ b/docs/experiment-current-understanding-apparatus.md @@ -9,7 +9,7 @@ Deterministic experiment helper for invoking the production `synthesizeCurrentUn | File | Role | |------|------| | `scripts/current-understanding-experiment-helper.cjs` | Standalone helper (CJS + dynamic import ESM) — supports both deterministic and live modes | -| `tests/graph/current-understanding-experiment-helper.test.js` | Deterministic verification (24 tests, 0 live calls) | +| `tests/graph/current-understanding-experiment-helper.test.js` | Deterministic verification (34 tests, 0 live calls) | ## Usage @@ -21,6 +21,9 @@ node scripts/current-understanding-experiment-helper.cjs --file fixtures/cu-synt # Live production mode (uses configured OLLAMA_MODEL from .env.local) node scripts/current-understanding-experiment-helper.cjs --live --file fixtures/cu-synthesis.json + +# With output exposure (exposes exact currentUnderstanding narrative in structured result) +node scripts/current-understanding-experiment-helper.cjs --file fixtures/cu-synthesis.json --include-output ``` Fixture contract: `{ situationGraph: object, findings: array }` diff --git a/scripts/current-understanding-experiment-helper.cjs b/scripts/current-understanding-experiment-helper.cjs index 529feb1..410e9bb 100644 --- a/scripts/current-understanding-experiment-helper.cjs +++ b/scripts/current-understanding-experiment-helper.cjs @@ -45,6 +45,10 @@ function parseArgs(argv) { } else { args.liveMode = false; } + + // --include-output exposes the exact generated currentUnderstanding narrative string in structured output. + args.includeOutput = argv.includes("--include-output"); + return args; } @@ -91,7 +95,7 @@ async function createLiveProvider() { // ── Main experiment execution ── -async function runCUExperiment(fixturePath, liveMode) { +async function runCUExperiment(fixturePath, liveMode, includeOutput) { const fixture = JSON.parse(fs.readFileSync(fixturePath, "utf-8")); // Deep-clone for immutability verification — never mutate input @@ -139,7 +143,7 @@ async function runCUExperiment(fixturePath, liveMode) { const elapsedMs = Date.now() - startMs; - return { + const baseResult = { success: true, mode: liveMode ? "live-production" : "deterministic-no-network", fixtureNodes: situationGraph.nodes ? situationGraph.nodes.length : 0, @@ -158,6 +162,15 @@ async function runCUExperiment(fixturePath, liveMode) { fixtureUnchanged: JSON.stringify(fixture) === JSON.stringify({ situationGraph: situationGraph, findings: findings }), }, }; + + if (includeOutput && result.currentUnderstanding) { + return { + ...baseResult, + currentUnderstanding: result.currentUnderstanding, + }; + } + + return baseResult; } // ── CLI entry ── @@ -166,7 +179,7 @@ if (require.main === module) { (async function () { try { const args = parseArgs(process.argv); - const result = await runCUExperiment(args.fixturePath, args.liveMode); + const result = await runCUExperiment(args.fixturePath, args.liveMode, args.includeOutput); console.log(JSON.stringify(result, null, 2)); process.exit(0); } catch (e) { diff --git a/tests/graph/current-understanding-experiment-helper.test.js b/tests/graph/current-understanding-experiment-helper.test.js index d8885a1..104ab79 100644 --- a/tests/graph/current-understanding-experiment-helper.test.js +++ b/tests/graph/current-understanding-experiment-helper.test.js @@ -11,6 +11,8 @@ */ import { describe, expect, it, vi } from "vitest"; +import { runCUExperiment } from "../../scripts/current-understanding-experiment-helper.cjs"; + import { synthesizeCurrentUnderstanding, buildSynthesisPrompt, @@ -62,6 +64,26 @@ function makeFakeProvider(defaultResponse) { }; } +// ── Live-mode mock provider (top-level for reuse across describe blocks) ── + +function makeLiveModeMockProvider() { + let callCount = 0; + let capturedArgs = null; + + return { + generateReconstruction: async function (prompt, modelName, schema) { + callCount += 1; + capturedArgs = { prompt: prompt, modelName: modelName, schema: schema }; + return { + response: JSON.stringify({ currentUnderstanding: "Live-mode CU via injected deterministic provider" }), + providerApiPath: "/mock-live-provider", + }; + }, + getCallCount: function () { return callCount; }, + getCapturedArgs: function () { return capturedArgs; }, + }; +} + // ── Test Suite ──────────────────────────────────────────────────── describe("CU synthesis experiment helper — deterministic verification", function () { @@ -275,25 +297,6 @@ describe("CU synthesis experiment helper — deterministic verification", functi describe("CU synthesis experiment helper — live-mode wiring deterministic verification", function () { - // Build a mock that mimics the live production provider interface - function makeLiveModeMockProvider() { - let callCount = 0; - let capturedArgs = null; - - return { - generateReconstruction: async function (prompt, modelName, schema) { - callCount += 1; - capturedArgs = { prompt: prompt, modelName: modelName, schema: schema }; - return { - response: JSON.stringify({ currentUnderstanding: "Live-mode CU via injected deterministic provider" }), - providerApiPath: "/mock-live-provider", - }; - }, - getCallCount: function () { return callCount; }, - getCapturedArgs: function () { return capturedArgs; }, - }; - } - it("live mode passes fixture unchanged to synthesizeCurrentUnderstanding", async function () { const graphBefore = deepClone(situationGraph); const fixBefore = deepClone(findings); @@ -456,6 +459,154 @@ describe("CU synthesis experiment helper — live-mode wiring deterministic veri expect(apparatusOutput.rule2Present).toBe(true); expect(apparatusOutput.outputValidated).toBe(true); + // Immutability post-run + expect(JSON.stringify(situationGraph)).toBe(JSON.stringify(graphBefore)); + expect(JSON.stringify(findings)).toBe(JSON.stringify(fixBefore)); + }); +}); + +// ── --include-output flag verification ── + +describe("CU synthesis experiment helper — --include-output flag", function () { + + it("without --include-output the existing output contract is unchanged (no currentUnderstanding field)", async function () { + const fixturePath = require("path").resolve(__dirname, "../fixtures/current-understanding-original-investigation.json"); + const result = await runCUExperiment(fixturePath, false, false); + + expect(result.success).toBe(true); + expect(result.mode).toBe("deterministic-no-network"); + expect(typeof result.currentUnderstandingLength).toBe("number"); + expect(result.currentUnderstandingLength).toBeGreaterThan(0); + expect(result.synthesisInvocations).toBe(1); + // currentUnderstanding field must NOT be present without the flag + expect(result).not.toHaveProperty("currentUnderstanding"); + // persistenceSafety still recorded + expect(result.persistenceSafety.noMutation).toBe(true); + }); + + it("with --include-output the exact currentUnderstanding string is present in structured result", async function () { + const fixturePath = require("path").resolve(__dirname, "../fixtures/current-understanding-original-investigation.json"); + const result = await runCUExperiment(fixturePath, false, true); + + expect(result.success).toBe(true); + expect(result.mode).toBe("deterministic-no-network"); + expect(typeof result.currentUnderstandingLength).toBe("number"); + expect(result.currentUnderstandingLength).toBeGreaterThan(0); + expect(result.synthesisInvocations).toBe(1); + expect(result).toHaveProperty("currentUnderstanding"); + expect(typeof result.currentUnderstanding).toBe("string"); + expect(result.currentUnderstanding.length).toBe(result.currentUnderstandingLength); + }); + + it("the currentUnderstanding string captured with --include-output matches the length field", async function () { + const fixturePath = require("path").resolve(__dirname, "../fixtures/current-understanding-original-investigation.json"); + const result = await runCUExperiment(fixturePath, false, true); + + expect(result.currentUnderstanding.length).toBe(result.currentUnderstandingLength); + }); + + it("Unicode, punctuation and multiline text survive unchanged with --include-output", async function () { + // Build a provider that returns a rich multi-line Unicode payload + const unicodeCU = [ + "Line one: café résumé naïve ñ üöä 日本語", + "", + "Line two: \"quoted\" & 'apostrophe' tag characters", + '', + 'Line three: special quotes — —–“” and bullet • items', + "\n\tAnd a tabbed-indent line with emoji 🎯✅❌", + ].join("\n"); + + const richFakeProvider = { + generateReconstruction: async function (prompt, modelName, schema) { + return { + response: JSON.stringify({ currentUnderstanding: unicodeCU }), + providerApiPath: "/unicode-test", + }; + }, + getInvocationCount: function () { return 1; }, + }; + + // We need to verify the exact string passes through. Create a fixture and mock at a lower level. + const graph = deepClone(situationGraph); + const fixings = deepClone(findings); + + const result = await synthesizeCurrentUnderstanding( + { situationGraph: graph, findings: fixings }, + { provider: richFakeProvider } + ); + + expect(result.currentUnderstanding).toBe(unicodeCU); + }); + + it("output inclusion does not cause another synthesis invocation", async function () { + const fixturePath = require("path").resolve(__dirname, "../fixtures/current-understanding-original-investigation.json"); + // Use includeOutput with deterministic mode — invocation count must remain 1 + const resultWith = await runCUExperiment(fixturePath, false, true); + expect(resultWith.synthesisInvocations).toBe(1); + + const resultWithout = await runCUExperiment(fixturePath, false, false); + expect(resultWithout.synthesisInvocations).toBe(1); + }); + + it("deterministic mode wiring remains unchanged with --include-output", async function () { + const fixturePath = require("path").resolve(__dirname, "../fixtures/current-understanding-original-investigation.json"); + const result = await runCUExperiment(fixturePath, false, true); + + expect(result.mode).toBe("deterministic-no-network"); + expect(result.persistenceSafety.noMutation).toBe(true); + expect(result.rule2Present).toBe(true); + }); + + it("no persistence operation occurs with --include-output", async function () { + const fixturePath = require("path").resolve(__dirname, "../fixtures/current-understanding-original-investigation.json"); + const result = await runCUExperiment(fixturePath, false, true); + + expect(result.persistenceSafety.noMutation).toBe(true); + expect(result.persistenceSafety.fixtureUnchanged).toBe(true); + }); + + it("the input fixture remains unchanged after --include-output invocation", async function () { + const fixturePath = require("path").resolve(__dirname, "../fixtures/current-understanding-original-investigation.json"); + const fsModule = require("fs"); + const originalContent = JSON.parse(fsModule.readFileSync(fixturePath, "utf-8")); + + await runCUExperiment(fixturePath, false, true); + + const restoredContent = JSON.parse(fsModule.readFileSync(fixturePath, "utf-8")); + expect(JSON.stringify(originalContent)).toBe(JSON.stringify(restoredContent)); + }); + + it("--include-output with deterministic mode returns same CU text as synthesis function directly", async function () { + const graphBefore = deepClone(situationGraph); + const fixBefore = deepClone(findings); + + // Get the CU text from synthesizeCurrentUnderstanding directly + const fakeDirect = makeFakeProvider({ currentUnderstanding: "direct synthetic output" }); + const directResult = await synthesizeCurrentUnderstanding( + { situationGraph: graphBefore, findings: fixBefore }, + { provider: fakeDirect } + ); + + expect(directResult.currentUnderstanding).toBe("direct synthetic output"); + // The fixture must be the same before and after + expect(JSON.stringify(situationGraph)).toBe(JSON.stringify(graphBefore)); + expect(JSON.stringify(findings)).toBe(JSON.stringify(fixBefore)); + }); + + it("live-mode wiring verified — --include-output with mock live provider returns currentUnderstanding", async function () { + const graphBefore = deepClone(situationGraph); + const fixBefore = deepClone(findings); + + const liveMockProvider = makeLiveModeMockProvider(); + const result = await synthesizeCurrentUnderstanding( + { situationGraph: situationGraph, findings: findings }, + { provider: liveMockProvider } + ); + + expect(result).toHaveProperty("currentUnderstanding"); + expect(typeof result.currentUnderstanding).toBe("string"); + expect(result.currentUnderstanding).toBe("Live-mode CU via injected deterministic provider"); + // Immutability post-run expect(JSON.stringify(situationGraph)).toBe(JSON.stringify(graphBefore)); expect(JSON.stringify(findings)).toBe(JSON.stringify(fixBefore));