test(confidence-engine): compose deep CU checkpoints
This commit is contained in:
@@ -14,7 +14,7 @@
|
|||||||
* 6. Absent updated graph produces null structural checkpoint
|
* 6. Absent updated graph produces null structural checkpoint
|
||||||
*/
|
*/
|
||||||
|
|
||||||
const { extractLiveExperimentCheckpoint, _extractReasoningState } = require("./live-update-experiment-helper.cjs");
|
const { extractLiveExperimentCheckpoint, _extractReasoningState, composeDeepInvestigationCheckpoint } = require("./live-update-experiment-helper.cjs");
|
||||||
|
|
||||||
// ---- deriveCategory kept in test because it is NOT exported from the helper ----
|
// ---- deriveCategory kept in test because it is NOT exported from the helper ----
|
||||||
// (It is part of the answer-meaning extraction infrastructure, not the checkpoint seam)
|
// (It is part of the answer-meaning extraction infrastructure, not the checkpoint seam)
|
||||||
@@ -136,8 +136,10 @@ eq(checkpoint.modelElapsedMs, 123, "modelElapsedMs value matches provider timing
|
|||||||
assert(typeof checkpoint.endToEndElapsedMs === "number", "endToEndElapsedMs is number");
|
assert(typeof checkpoint.endToEndElapsedMs === "number", "endToEndElapsedMs is number");
|
||||||
eq(checkpoint.endToEndElapsedMs, 456, "endToEndElapsedMs value matches wrapper timing");
|
eq(checkpoint.endToEndElapsedMs, 456, "endToEndElapsedMs value matches wrapper timing");
|
||||||
|
|
||||||
// 4. Findings count from input graph (not production path)
|
// 4. Findings count from input graph (INVALID — labels graph node count as findings)
|
||||||
eq(checkpoint.findingsCount.total, 2, "findings total matches input graph length");
|
// NOTE: This field is MISLABELED in extractLiveExperimentCheckpoint: it reads params.graph nodes length,
|
||||||
|
// not a canonical Findings collection. Use composeDeepInvestigationCheckpoint for accurate counts.
|
||||||
|
eq(checkpoint.findingsCount.total, 2, "findings total reflects input graph node count (mislabelled as findings)");
|
||||||
assert(checkpoint.findingsCount.eligible === "UNAVAILABLE", "eligible finding count is UNAVAILABLE");
|
assert(checkpoint.findingsCount.eligible === "UNAVAILABLE", "eligible finding count is UNAVAILABLE");
|
||||||
|
|
||||||
// 5. CU evidence propositions UNAVAILABLE
|
// 5. CU evidence propositions UNAVAILABLE
|
||||||
@@ -223,6 +225,25 @@ assert(absentCheckpoint.reasoningState === null, "reasoningState is null when ab
|
|||||||
var nullRs = _extractReasoningState(null);
|
var nullRs = _extractReasoningState(null);
|
||||||
assert(nullRs === null, "_extractReasoningState returns null for null input");
|
assert(nullRs === null, "_extractReasoningState returns null for null input");
|
||||||
|
|
||||||
|
// ---- COMPOSE DEEP INVESTIGATION CHECKPOINT (ITERATION 3 — NEW) ----
|
||||||
|
console.log("\n=== COMPOSED DEEP INVESTIGATION CHECKPOINT ===\n");
|
||||||
|
|
||||||
|
// verify the new composition function is exported and callable
|
||||||
|
assert(typeof composeDeepInvestigationCheckpoint === "function", "composeDeepInvestigationCheckpoint is exported as a function");
|
||||||
|
|
||||||
|
// Compose takes { situationGraph, previousQuestion, answer } — same contract as runLiveExperiment
|
||||||
|
// It mirrors production: episode done server seam → CU synthesis using updated graph + accumulated findings
|
||||||
|
|
||||||
|
// Verify exports match what composition needs — spot-check availability of canonical imports
|
||||||
|
assert(typeof extractLiveExperimentCheckpoint === "function", "extractLiveExperimentCheckpoint exported");
|
||||||
|
assert(typeof composeDeepInvestigationCheckpoint === "function", "composeDeepInvestigationCheckpoint exported alongside existing seam");
|
||||||
|
|
||||||
|
// The function is designed to be invoked with real Ollama config. Without .env.local,
|
||||||
|
// it will fail with a config assertion — which is expected and acceptable for this harness.
|
||||||
|
// It MUST NOT be called in deterministic test context (requires live model).
|
||||||
|
assert(composeDeepInvestigationCheckpoint.length >= 1, "composeDeepInvestigationCheckpoint accepts at least 1 parameter");
|
||||||
|
|
||||||
|
|
||||||
// ---- exit with failure count ----
|
// ---- exit with failure count ----
|
||||||
console.log("\n--- Total failures: " + failures + " ---\n");
|
console.log("\n--- Total failures: " + failures + " ---\n");
|
||||||
process.exit(failures > 0 ? 1 : 0);
|
process.exit(failures > 0 ? 1 : 0);
|
||||||
@@ -210,4 +210,116 @@ async function runLiveExperiment({ graph, previousQuestion, answer }) {
|
|||||||
return extractLiveExperimentCheckpoint(result, { graph, answer });
|
return extractLiveExperimentCheckpoint(result, { graph, answer });
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { runLiveExperiment, extractLiveExperimentCheckpoint, _extractReasoningState };
|
/**
|
||||||
|
* Compose the canonical post-episode deep-investigation checkpoint sequence:
|
||||||
|
* updateCase → synthesizeCurrentUnderstanding
|
||||||
|
*
|
||||||
|
* Mirrors the production flow in ScenarioForm.handleDoneForNowPromotion / executeEpisodeDone:
|
||||||
|
* 1. Call the episode done server seam (POST /api/cases/update, episodeMode=true)
|
||||||
|
* which updates the SituationGraph and accumulates Findings
|
||||||
|
* 2. Use the updated graph + accumulated findings to synthesize Current Understanding
|
||||||
|
* via the same seam as production (POST /api/cases/synthesis → synthesizeCurrentUnderstanding)
|
||||||
|
*
|
||||||
|
* Returns a combined checkpoint containing:
|
||||||
|
* - extractLiveExperimentCheckpoint(...) fields from the episode done result
|
||||||
|
* - currentUnderstanding from the synthesis result
|
||||||
|
* - findingsCheckpoint with accurate post-episode accumulated findings count
|
||||||
|
*/
|
||||||
|
async function composeDeepInvestigationCheckpoint({ situationGraph, previousQuestion, answer }) {
|
||||||
|
dotenv.config({ path: ".env.local" });
|
||||||
|
|
||||||
|
const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL");
|
||||||
|
const model = assertRequiredEnv("OLLAMA_MODEL");
|
||||||
|
|
||||||
|
if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
|
||||||
|
throw new Error(
|
||||||
|
`Live experiment harness refuses to use localhost fallback. ` +
|
||||||
|
`OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── Step 1: episode done (mirrors executeEpisodeDone server seam) ─── */
|
||||||
|
const { updateCase } = await import(
|
||||||
|
"../../lib/graph/orchestrator.js"
|
||||||
|
);
|
||||||
|
const { getProvider } = await import("../../lib/llm/provider.js");
|
||||||
|
|
||||||
|
const provider = getProvider();
|
||||||
|
let modelElapsedMs = null;
|
||||||
|
let providerGenerateCalls = 0;
|
||||||
|
|
||||||
|
const timedProvider = {
|
||||||
|
async generateReconstruction(prompt, modelName) {
|
||||||
|
providerGenerateCalls++;
|
||||||
|
const startedAt = Date.now();
|
||||||
|
const response = await provider.generateReconstruction(prompt, modelName);
|
||||||
|
modelElapsedMs = Date.now() - startedAt;
|
||||||
|
return response;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const endToEndStartedAt = Date.now();
|
||||||
|
const episodeResult = await updateCase(
|
||||||
|
{ situationGraph, previousQuestion, answer },
|
||||||
|
{ provider: timedProvider },
|
||||||
|
);
|
||||||
|
const endToEndElapsedMs = Date.now() - endToEndStartedAt;
|
||||||
|
|
||||||
|
/* ── Step 2: Current Understanding synthesis (mirrors production synthesizeFn) ─── */
|
||||||
|
const updatedGraph = episodeResult.updatedSituationGraph;
|
||||||
|
const accumulatedFindings = episodeResult.appendedFindings ?? [];
|
||||||
|
|
||||||
|
// Collect accumulated findings from the episode result for accurate counting
|
||||||
|
let postEpisodeFindings = Array.isArray(accumulatedFindings) ? [...accumulatedFindings] : [];
|
||||||
|
|
||||||
|
// Derive eligible count via the same filter the production seam uses
|
||||||
|
const { filterEligibleFindings } = await import("../../lib/graph/current-understanding-synthesis.js");
|
||||||
|
const eligibleCount = postEpisodeFindings.length > 0
|
||||||
|
? filterEligibleFindings(postEpisodeFindings).length
|
||||||
|
: 0;
|
||||||
|
|
||||||
|
let currentUnderstanding = null;
|
||||||
|
if (episodeResult.success && updatedGraph) {
|
||||||
|
// Reuse the same provider/model plumbing — no re-creation, just reuse
|
||||||
|
const { synthesizeCurrentUnderstanding } = await import("../../lib/graph/current-understanding-synthesis.js");
|
||||||
|
|
||||||
|
const synthesized = await synthesizeCurrentUnderstanding(
|
||||||
|
{ situationGraph: updatedGraph, findings: postEpisodeFindings },
|
||||||
|
{ provider: timedProvider, modelName: model },
|
||||||
|
);
|
||||||
|
|
||||||
|
currentUnderstanding = synthesized?.currentUnderstanding ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
/* ── CU synthesis result (NEW — mirrors production synthesizeFn) ─── */
|
||||||
|
currentUnderstanding,
|
||||||
|
|
||||||
|
/* ── Accurate post-episode findings count (FIX — uses actual accumulated Findings, not graph nodes) ─── */
|
||||||
|
findingsCheckpoint: {
|
||||||
|
total: postEpisodeFindings.length,
|
||||||
|
eligible: eligibleCount,
|
||||||
|
},
|
||||||
|
|
||||||
|
/* ── Timing (mirrors extractLiveExperimentCheckpoint) ─── */
|
||||||
|
modelElapsedMs,
|
||||||
|
endToEndElapsedMs,
|
||||||
|
providerGenerateCalls,
|
||||||
|
modelName: episodeResult.modelName ?? null,
|
||||||
|
|
||||||
|
/* ── Graph checkpoint (from production result) ─── */
|
||||||
|
graphCheckpoint: updatedGraph
|
||||||
|
? {
|
||||||
|
nodeCount: updatedGraph.nodes?.length ?? 0,
|
||||||
|
edgeCount: updatedGraph.edges?.length ?? 0,
|
||||||
|
resolvedNodeIds: updatedGraph.resolvedNodeIds ?? [],
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
|
||||||
|
/* ── Answer meaning (from production result) ─── */
|
||||||
|
userSupportedMeaning: episodeResult?.proposal?.answerMeaning?.userSupportedMeaning ?? null,
|
||||||
|
possibleInference: episodeResult?.proposal?.answerMeaning?.possibleInference ?? null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { runLiveExperiment, extractLiveExperimentCheckpoint, _extractReasoningState, composeDeepInvestigationCheckpoint };
|
||||||
|
|||||||
Reference in New Issue
Block a user