From 90649c04952a8107e897d70f300a617b81057768 Mon Sep 17 00:00:00 2001 From: robbond Date: Mon, 5 Oct 2026 06:44:29 +0100 Subject: [PATCH] test(confidence-engine): compose deep CU checkpoints --- ...date-experiment-helper-checkpoint.test.cjs | 27 ++++- tests/graph/live-update-experiment-helper.cjs | 114 +++++++++++++++++- 2 files changed, 137 insertions(+), 4 deletions(-) diff --git a/tests/graph/live-update-experiment-helper-checkpoint.test.cjs b/tests/graph/live-update-experiment-helper-checkpoint.test.cjs index 417b095..7b687dc 100644 --- a/tests/graph/live-update-experiment-helper-checkpoint.test.cjs +++ b/tests/graph/live-update-experiment-helper-checkpoint.test.cjs @@ -14,7 +14,7 @@ * 6. Absent updated graph produces null structural checkpoint */ -const { extractLiveExperimentCheckpoint, _extractReasoningState } = require("./live-update-experiment-helper.cjs"); +const { extractLiveExperimentCheckpoint, _extractReasoningState, composeDeepInvestigationCheckpoint } = require("./live-update-experiment-helper.cjs"); // ---- deriveCategory kept in test because it is NOT exported from the helper ---- // (It is part of the answer-meaning extraction infrastructure, not the checkpoint seam) @@ -136,8 +136,10 @@ eq(checkpoint.modelElapsedMs, 123, "modelElapsedMs value matches provider timing assert(typeof checkpoint.endToEndElapsedMs === "number", "endToEndElapsedMs is number"); eq(checkpoint.endToEndElapsedMs, 456, "endToEndElapsedMs value matches wrapper timing"); -// 4. Findings count from input graph (not production path) -eq(checkpoint.findingsCount.total, 2, "findings total matches input graph length"); +// 4. Findings count from input graph (INVALID — labels graph node count as findings) +// NOTE: This field is MISLABELED in extractLiveExperimentCheckpoint: it reads params.graph nodes length, +// not a canonical Findings collection. Use composeDeepInvestigationCheckpoint for accurate counts. +eq(checkpoint.findingsCount.total, 2, "findings total reflects input graph node count (mislabelled as findings)"); assert(checkpoint.findingsCount.eligible === "UNAVAILABLE", "eligible finding count is UNAVAILABLE"); // 5. CU evidence propositions UNAVAILABLE @@ -223,6 +225,25 @@ assert(absentCheckpoint.reasoningState === null, "reasoningState is null when ab var nullRs = _extractReasoningState(null); assert(nullRs === null, "_extractReasoningState returns null for null input"); +// ---- COMPOSE DEEP INVESTIGATION CHECKPOINT (ITERATION 3 — NEW) ---- +console.log("\n=== COMPOSED DEEP INVESTIGATION CHECKPOINT ===\n"); + +// verify the new composition function is exported and callable +assert(typeof composeDeepInvestigationCheckpoint === "function", "composeDeepInvestigationCheckpoint is exported as a function"); + +// Compose takes { situationGraph, previousQuestion, answer } — same contract as runLiveExperiment +// It mirrors production: episode done server seam → CU synthesis using updated graph + accumulated findings + +// Verify exports match what composition needs — spot-check availability of canonical imports +assert(typeof extractLiveExperimentCheckpoint === "function", "extractLiveExperimentCheckpoint exported"); +assert(typeof composeDeepInvestigationCheckpoint === "function", "composeDeepInvestigationCheckpoint exported alongside existing seam"); + +// The function is designed to be invoked with real Ollama config. Without .env.local, +// it will fail with a config assertion — which is expected and acceptable for this harness. +// It MUST NOT be called in deterministic test context (requires live model). +assert(composeDeepInvestigationCheckpoint.length >= 1, "composeDeepInvestigationCheckpoint accepts at least 1 parameter"); + + // ---- exit with failure count ---- console.log("\n--- Total failures: " + failures + " ---\n"); process.exit(failures > 0 ? 1 : 0); \ No newline at end of file diff --git a/tests/graph/live-update-experiment-helper.cjs b/tests/graph/live-update-experiment-helper.cjs index 3347681..dcb0254 100644 --- a/tests/graph/live-update-experiment-helper.cjs +++ b/tests/graph/live-update-experiment-helper.cjs @@ -210,4 +210,116 @@ async function runLiveExperiment({ graph, previousQuestion, answer }) { return extractLiveExperimentCheckpoint(result, { graph, answer }); } -module.exports = { runLiveExperiment, extractLiveExperimentCheckpoint, _extractReasoningState }; +/** + * Compose the canonical post-episode deep-investigation checkpoint sequence: + * updateCase → synthesizeCurrentUnderstanding + * + * Mirrors the production flow in ScenarioForm.handleDoneForNowPromotion / executeEpisodeDone: + * 1. Call the episode done server seam (POST /api/cases/update, episodeMode=true) + * which updates the SituationGraph and accumulates Findings + * 2. Use the updated graph + accumulated findings to synthesize Current Understanding + * via the same seam as production (POST /api/cases/synthesis → synthesizeCurrentUnderstanding) + * + * Returns a combined checkpoint containing: + * - extractLiveExperimentCheckpoint(...) fields from the episode done result + * - currentUnderstanding from the synthesis result + * - findingsCheckpoint with accurate post-episode accumulated findings count + */ +async function composeDeepInvestigationCheckpoint({ situationGraph, previousQuestion, answer }) { + dotenv.config({ path: ".env.local" }); + + const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL"); + const model = assertRequiredEnv("OLLAMA_MODEL"); + + if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") { + throw new Error( + `Live experiment harness refuses to use localhost fallback. ` + + `OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.` + ); + } + + /* ── Step 1: episode done (mirrors executeEpisodeDone server seam) ─── */ + const { updateCase } = await import( + "../../lib/graph/orchestrator.js" + ); + const { getProvider } = await import("../../lib/llm/provider.js"); + + const provider = getProvider(); + let modelElapsedMs = null; + let providerGenerateCalls = 0; + + const timedProvider = { + async generateReconstruction(prompt, modelName) { + providerGenerateCalls++; + const startedAt = Date.now(); + const response = await provider.generateReconstruction(prompt, modelName); + modelElapsedMs = Date.now() - startedAt; + return response; + }, + }; + + const endToEndStartedAt = Date.now(); + const episodeResult = await updateCase( + { situationGraph, previousQuestion, answer }, + { provider: timedProvider }, + ); + const endToEndElapsedMs = Date.now() - endToEndStartedAt; + + /* ── Step 2: Current Understanding synthesis (mirrors production synthesizeFn) ─── */ + const updatedGraph = episodeResult.updatedSituationGraph; + const accumulatedFindings = episodeResult.appendedFindings ?? []; + + // Collect accumulated findings from the episode result for accurate counting + let postEpisodeFindings = Array.isArray(accumulatedFindings) ? [...accumulatedFindings] : []; + + // Derive eligible count via the same filter the production seam uses + const { filterEligibleFindings } = await import("../../lib/graph/current-understanding-synthesis.js"); + const eligibleCount = postEpisodeFindings.length > 0 + ? filterEligibleFindings(postEpisodeFindings).length + : 0; + + let currentUnderstanding = null; + if (episodeResult.success && updatedGraph) { + // Reuse the same provider/model plumbing — no re-creation, just reuse + const { synthesizeCurrentUnderstanding } = await import("../../lib/graph/current-understanding-synthesis.js"); + + const synthesized = await synthesizeCurrentUnderstanding( + { situationGraph: updatedGraph, findings: postEpisodeFindings }, + { provider: timedProvider, modelName: model }, + ); + + currentUnderstanding = synthesized?.currentUnderstanding ?? null; + } + + return { + /* ── CU synthesis result (NEW — mirrors production synthesizeFn) ─── */ + currentUnderstanding, + + /* ── Accurate post-episode findings count (FIX — uses actual accumulated Findings, not graph nodes) ─── */ + findingsCheckpoint: { + total: postEpisodeFindings.length, + eligible: eligibleCount, + }, + + /* ── Timing (mirrors extractLiveExperimentCheckpoint) ─── */ + modelElapsedMs, + endToEndElapsedMs, + providerGenerateCalls, + modelName: episodeResult.modelName ?? null, + + /* ── Graph checkpoint (from production result) ─── */ + graphCheckpoint: updatedGraph + ? { + nodeCount: updatedGraph.nodes?.length ?? 0, + edgeCount: updatedGraph.edges?.length ?? 0, + resolvedNodeIds: updatedGraph.resolvedNodeIds ?? [], + } + : null, + + /* ── Answer meaning (from production result) ─── */ + userSupportedMeaning: episodeResult?.proposal?.answerMeaning?.userSupportedMeaning ?? null, + possibleInference: episodeResult?.proposal?.answerMeaning?.possibleInference ?? null, + }; +} + +module.exports = { runLiveExperiment, extractLiveExperimentCheckpoint, _extractReasoningState, composeDeepInvestigationCheckpoint };