test(confidence-engine): compose deep CU checkpoints
This commit is contained in:
@@ -210,4 +210,116 @@ async function runLiveExperiment({ graph, previousQuestion, answer }) {
|
||||
return extractLiveExperimentCheckpoint(result, { graph, answer });
|
||||
}
|
||||
|
||||
module.exports = { runLiveExperiment, extractLiveExperimentCheckpoint, _extractReasoningState };
|
||||
/**
|
||||
* Compose the canonical post-episode deep-investigation checkpoint sequence:
|
||||
* updateCase → synthesizeCurrentUnderstanding
|
||||
*
|
||||
* Mirrors the production flow in ScenarioForm.handleDoneForNowPromotion / executeEpisodeDone:
|
||||
* 1. Call the episode done server seam (POST /api/cases/update, episodeMode=true)
|
||||
* which updates the SituationGraph and accumulates Findings
|
||||
* 2. Use the updated graph + accumulated findings to synthesize Current Understanding
|
||||
* via the same seam as production (POST /api/cases/synthesis → synthesizeCurrentUnderstanding)
|
||||
*
|
||||
* Returns a combined checkpoint containing:
|
||||
* - extractLiveExperimentCheckpoint(...) fields from the episode done result
|
||||
* - currentUnderstanding from the synthesis result
|
||||
* - findingsCheckpoint with accurate post-episode accumulated findings count
|
||||
*/
|
||||
async function composeDeepInvestigationCheckpoint({ situationGraph, previousQuestion, answer }) {
|
||||
dotenv.config({ path: ".env.local" });
|
||||
|
||||
const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL");
|
||||
const model = assertRequiredEnv("OLLAMA_MODEL");
|
||||
|
||||
if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
|
||||
throw new Error(
|
||||
`Live experiment harness refuses to use localhost fallback. ` +
|
||||
`OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.`
|
||||
);
|
||||
}
|
||||
|
||||
/* ── Step 1: episode done (mirrors executeEpisodeDone server seam) ─── */
|
||||
const { updateCase } = await import(
|
||||
"../../lib/graph/orchestrator.js"
|
||||
);
|
||||
const { getProvider } = await import("../../lib/llm/provider.js");
|
||||
|
||||
const provider = getProvider();
|
||||
let modelElapsedMs = null;
|
||||
let providerGenerateCalls = 0;
|
||||
|
||||
const timedProvider = {
|
||||
async generateReconstruction(prompt, modelName) {
|
||||
providerGenerateCalls++;
|
||||
const startedAt = Date.now();
|
||||
const response = await provider.generateReconstruction(prompt, modelName);
|
||||
modelElapsedMs = Date.now() - startedAt;
|
||||
return response;
|
||||
},
|
||||
};
|
||||
|
||||
const endToEndStartedAt = Date.now();
|
||||
const episodeResult = await updateCase(
|
||||
{ situationGraph, previousQuestion, answer },
|
||||
{ provider: timedProvider },
|
||||
);
|
||||
const endToEndElapsedMs = Date.now() - endToEndStartedAt;
|
||||
|
||||
/* ── Step 2: Current Understanding synthesis (mirrors production synthesizeFn) ─── */
|
||||
const updatedGraph = episodeResult.updatedSituationGraph;
|
||||
const accumulatedFindings = episodeResult.appendedFindings ?? [];
|
||||
|
||||
// Collect accumulated findings from the episode result for accurate counting
|
||||
let postEpisodeFindings = Array.isArray(accumulatedFindings) ? [...accumulatedFindings] : [];
|
||||
|
||||
// Derive eligible count via the same filter the production seam uses
|
||||
const { filterEligibleFindings } = await import("../../lib/graph/current-understanding-synthesis.js");
|
||||
const eligibleCount = postEpisodeFindings.length > 0
|
||||
? filterEligibleFindings(postEpisodeFindings).length
|
||||
: 0;
|
||||
|
||||
let currentUnderstanding = null;
|
||||
if (episodeResult.success && updatedGraph) {
|
||||
// Reuse the same provider/model plumbing — no re-creation, just reuse
|
||||
const { synthesizeCurrentUnderstanding } = await import("../../lib/graph/current-understanding-synthesis.js");
|
||||
|
||||
const synthesized = await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: updatedGraph, findings: postEpisodeFindings },
|
||||
{ provider: timedProvider, modelName: model },
|
||||
);
|
||||
|
||||
currentUnderstanding = synthesized?.currentUnderstanding ?? null;
|
||||
}
|
||||
|
||||
return {
|
||||
/* ── CU synthesis result (NEW — mirrors production synthesizeFn) ─── */
|
||||
currentUnderstanding,
|
||||
|
||||
/* ── Accurate post-episode findings count (FIX — uses actual accumulated Findings, not graph nodes) ─── */
|
||||
findingsCheckpoint: {
|
||||
total: postEpisodeFindings.length,
|
||||
eligible: eligibleCount,
|
||||
},
|
||||
|
||||
/* ── Timing (mirrors extractLiveExperimentCheckpoint) ─── */
|
||||
modelElapsedMs,
|
||||
endToEndElapsedMs,
|
||||
providerGenerateCalls,
|
||||
modelName: episodeResult.modelName ?? null,
|
||||
|
||||
/* ── Graph checkpoint (from production result) ─── */
|
||||
graphCheckpoint: updatedGraph
|
||||
? {
|
||||
nodeCount: updatedGraph.nodes?.length ?? 0,
|
||||
edgeCount: updatedGraph.edges?.length ?? 0,
|
||||
resolvedNodeIds: updatedGraph.resolvedNodeIds ?? [],
|
||||
}
|
||||
: null,
|
||||
|
||||
/* ── Answer meaning (from production result) ─── */
|
||||
userSupportedMeaning: episodeResult?.proposal?.answerMeaning?.userSupportedMeaning ?? null,
|
||||
possibleInference: episodeResult?.proposal?.answerMeaning?.possibleInference ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { runLiveExperiment, extractLiveExperimentCheckpoint, _extractReasoningState, composeDeepInvestigationCheckpoint };
|
||||
|
||||
Reference in New Issue
Block a user