test(confidence-engine): compose deep CU checkpoints

2026-10-05 06:44:29 +01:00
parent 72b10d0050
commit 90649c0495
2 changed files with 137 additions and 4 deletions
@@ -14,7 +14,7 @@
 *   6. Absent updated graph produces null structural checkpoint
 */

-const { extractLiveExperimentCheckpoint, _extractReasoningState } = require("./live-update-experiment-helper.cjs");
+const { extractLiveExperimentCheckpoint, _extractReasoningState, composeDeepInvestigationCheckpoint } = require("./live-update-experiment-helper.cjs");

 // ---- deriveCategory kept in test because it is NOT exported from the helper ----
 // (It is part of the answer-meaning extraction infrastructure, not the checkpoint seam)
@@ -136,8 +136,10 @@ eq(checkpoint.modelElapsedMs, 123, "modelElapsedMs value matches provider timing
 assert(typeof checkpoint.endToEndElapsedMs === "number", "endToEndElapsedMs is number");
 eq(checkpoint.endToEndElapsedMs, 456, "endToEndElapsedMs value matches wrapper timing");

-// 4. Findings count from input graph (not production path)
-eq(checkpoint.findingsCount.total, 2, "findings total matches input graph length");
+// 4. Findings count from input graph (INVALID — labels graph node count as findings)
+//    NOTE: This field is MISLABELED in extractLiveExperimentCheckpoint: it reads params.graph nodes length,
+//    not a canonical Findings collection. Use composeDeepInvestigationCheckpoint for accurate counts.
+eq(checkpoint.findingsCount.total, 2, "findings total reflects input graph node count (mislabelled as findings)");
 assert(checkpoint.findingsCount.eligible === "UNAVAILABLE", "eligible finding count is UNAVAILABLE");

 // 5. CU evidence propositions UNAVAILABLE
@@ -223,6 +225,25 @@ assert(absentCheckpoint.reasoningState === null, "reasoningState is null when ab
 var nullRs = _extractReasoningState(null);
 assert(nullRs === null, "_extractReasoningState returns null for null input");

+// ---- COMPOSE DEEP INVESTIGATION CHECKPOINT (ITERATION 3 — NEW) ----
+console.log("\n=== COMPOSED DEEP INVESTIGATION CHECKPOINT ===\n");
+
+// verify the new composition function is exported and callable
+assert(typeof composeDeepInvestigationCheckpoint === "function", "composeDeepInvestigationCheckpoint is exported as a function");
+
+// Compose takes { situationGraph, previousQuestion, answer } — same contract as runLiveExperiment
+// It mirrors production: episode done server seam → CU synthesis using updated graph + accumulated findings
+
+// Verify exports match what composition needs — spot-check availability of canonical imports
+assert(typeof extractLiveExperimentCheckpoint === "function", "extractLiveExperimentCheckpoint exported");
+assert(typeof composeDeepInvestigationCheckpoint === "function", "composeDeepInvestigationCheckpoint exported alongside existing seam");
+
+// The function is designed to be invoked with real Ollama config. Without .env.local,
+// it will fail with a config assertion — which is expected and acceptable for this harness.
+// It MUST NOT be called in deterministic test context (requires live model).
+assert(composeDeepInvestigationCheckpoint.length >= 1, "composeDeepInvestigationCheckpoint accepts at least 1 parameter");
+
+
 // ---- exit with failure count ----
 console.log("\n--- Total failures: " + failures + " ---\n");
 process.exit(failures > 0 ? 1 : 0);
@@ -210,4 +210,116 @@ async function runLiveExperiment({ graph, previousQuestion, answer }) {
  return extractLiveExperimentCheckpoint(result, { graph, answer });
 }

-module.exports = { runLiveExperiment, extractLiveExperimentCheckpoint, _extractReasoningState };
+/**
+ * Compose the canonical post-episode deep-investigation checkpoint sequence:
+ *   updateCase → synthesizeCurrentUnderstanding
+ *
+ * Mirrors the production flow in ScenarioForm.handleDoneForNowPromotion / executeEpisodeDone:
+ *   1. Call the episode done server seam (POST /api/cases/update, episodeMode=true)
+ *      which updates the SituationGraph and accumulates Findings
+ *   2. Use the updated graph + accumulated findings to synthesize Current Understanding
+ *      via the same seam as production (POST /api/cases/synthesis → synthesizeCurrentUnderstanding)
+ *
+ * Returns a combined checkpoint containing:
+ *   - extractLiveExperimentCheckpoint(...) fields from the episode done result
+ *   - currentUnderstanding from the synthesis result
+ *   - findingsCheckpoint with accurate post-episode accumulated findings count
+ */
+async function composeDeepInvestigationCheckpoint({ situationGraph, previousQuestion, answer }) {
+  dotenv.config({ path: ".env.local" });
+
+  const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL");
+  const model   = assertRequiredEnv("OLLAMA_MODEL");
+
+  if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
+    throw new Error(
+      `Live experiment harness refuses to use localhost fallback. ` +
+      `OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.`
+    );
+  }
+
+  /* ── Step 1: episode done (mirrors executeEpisodeDone server seam) ─── */
+  const { updateCase } = await import(
+    "../../lib/graph/orchestrator.js"
+  );
+  const { getProvider } = await import("../../lib/llm/provider.js");
+
+  const provider = getProvider();
+  let modelElapsedMs = null;
+  let providerGenerateCalls = 0;
+
+  const timedProvider = {
+    async generateReconstruction(prompt, modelName) {
+      providerGenerateCalls++;
+      const startedAt = Date.now();
+      const response = await provider.generateReconstruction(prompt, modelName);
+      modelElapsedMs = Date.now() - startedAt;
+      return response;
+    },
+  };
+
+  const endToEndStartedAt = Date.now();
+  const episodeResult = await updateCase(
+    { situationGraph, previousQuestion, answer },
+    { provider: timedProvider },
+  );
+  const endToEndElapsedMs = Date.now() - endToEndStartedAt;
+
+  /* ── Step 2: Current Understanding synthesis (mirrors production synthesizeFn) ─── */
+  const updatedGraph   = episodeResult.updatedSituationGraph;
+  const accumulatedFindings = episodeResult.appendedFindings ?? [];
+
+  // Collect accumulated findings from the episode result for accurate counting
+  let postEpisodeFindings = Array.isArray(accumulatedFindings) ? [...accumulatedFindings] : [];
+
+  // Derive eligible count via the same filter the production seam uses
+  const { filterEligibleFindings } = await import("../../lib/graph/current-understanding-synthesis.js");
+  const eligibleCount = postEpisodeFindings.length > 0
+    ? filterEligibleFindings(postEpisodeFindings).length
+    : 0;
+
+  let currentUnderstanding = null;
+  if (episodeResult.success && updatedGraph) {
+    // Reuse the same provider/model plumbing — no re-creation, just reuse
+    const { synthesizeCurrentUnderstanding } = await import("../../lib/graph/current-understanding-synthesis.js");
+
+    const synthesized = await synthesizeCurrentUnderstanding(
+      { situationGraph: updatedGraph, findings: postEpisodeFindings },
+      { provider: timedProvider, modelName: model },
+    );
+
+    currentUnderstanding = synthesized?.currentUnderstanding ?? null;
+  }
+
+  return {
+    /* ── CU synthesis result (NEW — mirrors production synthesizeFn) ─── */
+    currentUnderstanding,
+
+    /* ── Accurate post-episode findings count (FIX — uses actual accumulated Findings, not graph nodes) ─── */
+    findingsCheckpoint: {
+      total: postEpisodeFindings.length,
+      eligible: eligibleCount,
+    },
+
+    /* ── Timing (mirrors extractLiveExperimentCheckpoint) ─── */
+    modelElapsedMs,
+    endToEndElapsedMs,
+    providerGenerateCalls,
+    modelName: episodeResult.modelName ?? null,
+
+    /* ── Graph checkpoint (from production result) ─── */
+    graphCheckpoint: updatedGraph
+      ? {
+          nodeCount:  updatedGraph.nodes?.length   ?? 0,
+          edgeCount:  updatedGraph.edges?.length   ?? 0,
+          resolvedNodeIds: updatedGraph.resolvedNodeIds ?? [],
+        }
+      : null,
+
+    /* ── Answer meaning (from production result) ─── */
+    userSupportedMeaning: episodeResult?.proposal?.answerMeaning?.userSupportedMeaning ?? null,
+    possibleInference:    episodeResult?.proposal?.answerMeaning?.possibleInference     ?? null,
+  };
+}
+
+module.exports = { runLiveExperiment, extractLiveExperimentCheckpoint, _extractReasoningState, composeDeepInvestigationCheckpoint };