/** * Canonical live-update experiment harness. * * Loads .env.local, validates Ollama configuration, invokes updateCase() * through the real production path, and captures the standard reasoning * checkpoints needed by reasoning experiments. * * Usage (from a test file): * const { runLiveExperiment } = require("./tests/graph/live-update-experiment-helper.cjs"); * * const result = await runLiveExperiment({ * graph: /* SituationGraph fixture *\/, * previousQuestion: "Is risk a hard constraint?", * answer: "Risk matters more to me.", * }); */ const dotenv = require("dotenv"); dotenv.config({ path: ".env.local" }); // Re-export production helper for apparatus use — avoids fabricating Finding propositions. const { deriveFindingsFromContributions } = require("../../lib/graph/finding-helpers.js"); function assertRequiredEnv(name) { const value = process.env[name]; if (!value) { throw new Error( `Live experiment requires ${name}. Set it in .env.local.\n` + `Found: OLLAMA_BASE_URL=${process.env.OLLAMA_BASE_URL ?? "(missing)"}, ` + `OLLAMA_MODEL=${process.env.OLLAMA_MODEL ?? "(missing)"}` ); } return value; } /** * Extract checkpoint data from a raw updateCase() result object. * * Pure function — no side effects, no I/O. Intended for use both by * `runLiveExperiment()` (apparatus wrapper) and by deterministic tests * that pass mock results directly. * * @param {object} result — raw result from updateCase() * @param {object} params — { graph?, answer? } * - graph: SituationGraph input fixture (for findingsCount total derived from input) * - answer: previous question text for deriveCategory fallback * @returns {object} checkpoint object per spec contract */ function extractLiveExperimentCheckpoint(result, params) { const answerMeaning = result?.proposal?.answerMeaning || {}; // Graph structural checkpoint (available from updatedSituationGraph only when present) const updatedGraph = result?.updatedSituationGraph; const graphCheckpoint = updatedGraph ? { nodeCount: updatedGraph.nodes?.length ?? 0, edgeCount: updatedGraph.edges?.length ?? 0, resolvedNodeIds: updatedGraph.resolvedNodeIds ?? [], } : null; return { // ── timing (available via wrapper) ──────────────────── modelElapsedMs: result?.modelElapsedMs !== undefined ? result.modelElapsedMs : null, endToEndElapsedMs: result?.endToEndElapsedMs !== undefined ? result.endToEndElapsedMs : null, providerGenerateCalls: result != null && result.providerGenerateCalls != null ? result.providerGenerateCalls : 0, modelName: result?.modelName || null, // ── answer meaning extraction ───────────────────────── userSupportedMeaning: answerMeaning.userSupportedMeaning ?? null, possibleInference: answerMeaning.possibleInference ?? null, rawAnswerCategory: _deriveCategory(params?.answer), proposedMeaningCategory: _deriveCategory(answerMeaning.userSupportedMeaning || ""), // ── validation / guard checkpoints ─────────────────── proposalValidation: result?.proposalValidation ?? { success: false, errors: [] }, compatibilityGuard: result?.compatibilityApplied !== undefined ? { passed: result.compatibilityApplied, warnings: [], violations: [] } : { passed: false, warnings: [], violations: [] }, // ── graph mutation checkpoint (from appliedGraph) ──── graphMutation: result?.appliedGraph ? { nodes: result.appliedGraph.nodes ?? null, edges: result.appliedGraph.edges ?? null } : null, // ── graph structural checkpoint (available from updatedSituationGraph) ─ graphCheckpoint, // ── findings checkpoint (from input graph fixture, not production path) ─ findingsCount: { total: (params?.graph && Array.isArray(params.graph)) ? params.graph.length : (params?.graph && params.graph.nodes != null) ? params.graph.nodes.length : 0, eligible: "UNAVAILABLE", // not canonically observable without production contract change }, // ── CU narrative (NOT captured — synthesis is a separate seam from updateCase()) ─ currentUnderstanding: "UNAVAILABLE", // ── candidate evidence (UNAVAILABLE without changing production code) ──────── eligibleFindingPropositions: "UNAVAILABLE", // ── helper internals (for diagnostic / test verification) ──────────────────── selectedQuestion: result?.proposal?.selectedQuestion ?? null, behaviourSelection: result?.behaviourSelection ?? null, reasoningState: _extractReasoningState(result), }; } /** * Minimal deterministic category derivation from free-text (mirrors the production pipeline). */ function _deriveCategory(text) { if (!text || !text.trim()) return "none"; const lower = text.toLowerCase(); // Conditional patterns if (/normally\s+(?:avoid|skip|not\s+take|would\s+n't|can\'t)/i.test(lower) && /(might|could|would\s+(?:accept|allow|take|do))/i.test(lower)) { return "conditional_tradeoff"; } // Contrast / but patterns if (/but\s+i?\s*(don\'t|cannot|can\'t|won\'t|will\s+not)/i.test(lower)) { return "qualified_support"; } // Explicit hard constraint if (/(hard\s+constraint|must\s+(?:not|never|always)|can\'?\s*t(?:o)\s*(?:not|be\s+able\s+to)|absolutely\s+cannot)/i.test(lower)) { return "hard_constraint"; } // Strong preference / must positive if (/(must\s+(?:have|do|get)|absolutely\s+(?:need|require)|cannot\s+proceed\s+without)/i.test(lower)) { return "strong_preference"; } // Relative importance if (/more\s+to\s+me|matters\s+more|higher\s+priority|top\s*priority/i.test(lower)) { return "relative_importance"; } // Support / contraindicate if (/support(s)?\b|confirm(s)?\b|validates?\b/i.test(lower)) { return "supports_decision"; } if (/contradicts?\b|against\s+it\b|i\s*don\'?\s*t\s*(?:think\s+so|agree)\b/i.test(lower)) { return "contradicts_decision"; } return "cannot_determine"; } /** * Extract reasoning state from a raw updateCase() result object. * Exported for deterministic test verification of preserved behaviour. */ function _extractReasoningState(result) { const rs = result?.reasoningState; if (!rs) return null; return { turnCount: rs.turnCount ?? null, health: rs.health ?? null, phase: rs.phase ?? null, progress: rs.progress ?? null, unknownStatuses: (rs.unknownStatuses || []).map(s => s.id + ":" + s.status), }; } /** * Run one live experiment through the real updateCase() production path. */ async function runLiveExperiment({ graph, previousQuestion, answer, findings = [] }) { const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL"); const model = assertRequiredEnv("OLLAMA_MODEL"); if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") { throw new Error( `Live experiment harness refuses to use localhost fallback. ` + `OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.` ); } const { updateCase } = await import( "../../lib/graph/orchestrator.js" ); const { getProvider } = await import("../../lib/llm/provider.js"); const provider = getProvider(); let modelElapsedMs = null; let providerGenerateCalls = 0; const timedProvider = { async generateReconstruction(prompt, modelName, outputSchema) { providerGenerateCalls++; const startedAt = Date.now(); const response = await provider.generateReconstruction(prompt, modelName, outputSchema); modelElapsedMs = Date.now() - startedAt; return response; }, }; const endToEndStartedAt = Date.now(); const result = await updateCase( { situationGraph: graph, previousQuestion, answer }, { provider: timedProvider }, ); const endToEndElapsedMs = Date.now() - endToEndStartedAt; // ── accumulated state for longitudinal multi-episode use ─── const updatedGraph = result?.updatedSituationGraph ?? null; const postEpisodeFs = Array.isArray(result?.findings) ? [...result.findings] : []; return { /* ── CU narrative (UNAVAILABLE — synthesis is a separate seam from updateCase()) ─── */ currentUnderstanding: "UNAVAILABLE", /* ── updatedSituationGraph (NEW for longitudinal state feeding) ─── */ updatedSituationGraph: updatedGraph, /* ── postEpisodeFindings (NEW — accumulated Findings exposed for Episode N+1) ─── */ postEpisodeFindings: postEpisodeFs, /* ── graph checkpoint (from production result) ─── */ graphCheckpoint: updatedGraph ? { nodeCount: updatedGraph.nodes?.length ?? 0, edgeCount: updatedGraph.edges?.length ?? 0, resolvedNodeIds: updatedGraph.resolvedNodeIds ?? [], } : { nodeCount: 0, edgeCount: 0, resolvedNodeIds: [] }, /* ── findings checkpoint (count from accumulated Findings) ─── */ findingsCheckpoint: { total: postEpisodeFs.length, eligible: "UNAVAILABLE", }, /* ── Timing ─── */ modelElapsedMs: result?.modelElapsedMs !== undefined ? result.modelElapsedMs : null, endToEndElapsedMs: result?.endToEndElapsedMs !== undefined ? result.endToEndElapsedMs : null, providerGenerateCalls: result != null && result.providerGenerateCalls != null ? result.providerGenerateCalls : 0, modelName: result?.modelName || null, /* ── answer meaning (from production result) ─── */ userSupportedMeaning: result?.proposal?.answerMeaning?.userSupportedMeaning ?? null, possibleInference: result?.proposal?.answerMeaning?.possibleInference ?? null, /* ── helper internals (for diagnostic / test verification) ─── */ selectedQuestion: result?.proposal?.selectedQuestion ?? null, behaviourSelection: result?.behaviourSelection ?? null, reasoningState: _extractReasoningState(result), }; } /** * Compose the canonical post-episode deep-investigation checkpoint sequence: * updateCase → synthesizeCurrentUnderstanding * * Mirrors the production flow in ScenarioForm.handleDoneForNowPromotion / executeEpisodeDone: * 1. Call the episode done server seam (POST /api/cases/update, episodeMode=true) * which updates the SituationGraph and accumulates Findings * 2. Use the updated graph + accumulated findings to synthesize Current Understanding * via the same seam as production (POST /api/cases/synthesis → synthesizeCurrentUnderstanding) * * Returns a combined checkpoint containing: * - extractLiveExperimentCheckpoint(...) fields from the episode done result * - currentUnderstanding from the synthesis result * - findingsCheckpoint with accurate post-episode accumulated findings count */ /** * Compose the canonical post-episode deep-investigation checkpoint sequence. * Uses production-focused deconstruct for Contribution derivation (not synthetic raw-answer path). */ async function composeDeepInvestigationCheckpoint({ situationGraph, previousQuestion, answer, findings = [], contributions = [], targetNodeId, episodeMode }) { dotenv.config({ path: ".env.local" }); const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL"); const model = assertRequiredEnv("OLLAMA_MODEL"); if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") { throw new Error( `Live experiment harness refuses to use localhost fallback. ` + `OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.` ); } /* ── Step 1: episode done (mirrors executeEpisodeDone server seam) ─── */ const { updateCase, reconsiderCompletedEpisode } = await import( "../../lib/graph/orchestrator.js" ); const { getProvider } = await import("../../lib/llm/provider.js"); const provider = getProvider(); let modelElapsedMs = null; let providerGenerateCalls = 0; const timedProvider = { async generateReconstruction(prompt, modelName, outputSchema) { providerGenerateCalls++; const startedAt = Date.now(); const response = await provider.generateReconstruction(prompt, modelName, outputSchema); modelElapsedMs = Date.now() - startedAt; return response; }, }; const endToEndStartedAt = Date.now(); let episodeResult; /* ── Production-equivalent focused-episode seam: targetNodeId triggers the same path as executeEpisodeDone → handleEpisodeMode ─── */ /* ── Canonical deconstruction: use focusedDeconstruct for Contribution derivation ─── */ const effectiveContributions = contributions.length > 0 ? [...contributions] : []; let newFindingsFromContributions = []; if (effectiveContributions.length === 0 && answer && targetNodeId) { const contribSeq = effectiveContributions.length + 1; /* ── Load production deconstruction seam (dynamic import to resolve ESM path aliases) ─── */ const { focusedDeconstruct } = await import("../../lib/graph/focused-investigation.js"); // Extract node context for deconstruction prompt const targetNode = situationGraph.nodes.find(n => n.id === targetNodeId); const targetLabel = targetNode?.label ?? ""; const targetDescription = targetNode?.description ?? ""; const centralStatement = situationGraph.centralStatement ?? ""; // Call production-focused deconstruct — exactly one provider generation const focused = await focusedDeconstruct({ provider: timedProvider, modelName: model, targetNodeId, targetLabel, targetDescription, centralStatement, question: previousQuestion ?? "", answer, }); if (!focused.success) { throw new Error(`focusedDeconstruct failed for episode ${contribSeq}: ${JSON.stringify(focused.errors)}`); } // Map deconstruction result → canonical Contribution shape const derivedContrib = { id: `contrib-ep-${String(contribSeq).padStart(4, "0")}`, targetNodeId: focused.targetNodeId ?? targetNodeId, question: previousQuestion ?? "", answer: answer, sequence: contribSeq, observations: focused.observations, uncertainties: focused.uncertainties, assumptions: focused.assumptions, relationships: focused.relationships, possibleFollowUpQuestions: focused.possibleFollowUpQuestions, }; effectiveContributions.push(derivedContrib); // Derive canonical Findings from the deconstruction-derived Contribution (production seam) newFindingsFromContributions = deriveFindingsFromContributions([derivedContrib]).findings; } if (episodeMode === true && targetNodeId) { // Mirrors produceEpisodeAwareGraph: prepareCompletedEpisode → reconsiderCompletedEpisode → applyValidatedProposal const { prepareCompletedEpisode } = await import("../../lib/graph/episode-preparation.js"); const { applyValidatedProposal } = await import("../../lib/graph/apply-proposal.js"); const prepared = prepareCompletedEpisode({ situationGraph, targetNodeId, contributions: effectiveContributions, findings, }); if (!prepared?.turns?.length && !prepared?.eligibleCanonicalFindings?.length) { throw new Error("no_episodic_content: prepared episode has no turns or eligible findings"); } const reasoning = await reconsiderCompletedEpisode(prepared); if (!reasoning.success) { throw new Error(`reconsiderCompletedEpisode failed: ${reasoning.error}`); } const application = await applyValidatedProposal({ situationGraph, proposal: reasoning.proposal, evidenceContext: { isCompletedEpisode: true, episodeEvidence: prepared, }, }); if (!application.success) { throw new Error(`applyValidatedProposal failed: ${application.error}`); } const resolvedNodeIds = new Set(application.updatedSituationGraph.resolvedNodeIds ?? []); resolvedNodeIds.add(targetNodeId); episodeResult = { success: true, updatedSituationGraph: { ...application.updatedSituationGraph, resolvedNodeIds: [...resolvedNodeIds], }, appendedFindings: [], // no regular update findings in episode path proposal: reasoning.proposal, }; } else { const result = await updateCase( { situationGraph, previousQuestion, answer }, { provider: timedProvider }, ); episodeResult = result; } const endToEndElapsedMs = Date.now() - endToEndStartedAt; /* ── Step 2: Current Understanding synthesis (mirrors production synthesizeFn) ─── */ const updatedGraph = episodeResult.updatedSituationGraph; // Accumulate prior findings + newly derived Findings (from new Contribution) for CU synthesis const priorFindings = Array.isArray(findings) ? [...findings] : []; const accumulatedEpisodes = episodeResult.appendedFindings ?? []; const postEpisodeFindings = [...priorFindings, ...newFindingsFromContributions, ...accumulatedEpisodes]; let currentUnderstanding = null; let synthesisErrorDiagnostic = null; if (episodeResult.success && updatedGraph) { // Reuse the same provider/model plumbing — no re-creation, just reuse const { synthesizeCurrentUnderstanding } = await import("../../lib/graph/current-understanding-synthesis.js"); try { const synthesized = await synthesizeCurrentUnderstanding( { situationGraph: updatedGraph, findings: postEpisodeFindings }, { provider: timedProvider, modelName: model }, ); currentUnderstanding = synthesized?.currentUnderstanding ?? null; } catch (synthErr) { // Expose the safe structural diagnostic from synthesis validation failure if (synthErr?.sanitizedDiagnostic) { synthesisErrorDiagnostic = synthErr.sanitizedDiagnostic; } currentUnderstanding = null; } } // Consistent graphCheckpoint for both success and failure paths const graphCheckpoint = updatedGraph ? { nodeCount: updatedGraph.nodes?.length ?? 0, edgeCount: updatedGraph.edges?.length ?? 0, resolvedNodeIds: updatedGraph.resolvedNodeIds ?? [], } : { nodeCount: 0, edgeCount: 0, resolvedNodeIds: [] }; // Accurate post-episode accumulated findings count const { filterEligibleFindings } = await import("../../lib/graph/current-understanding-synthesis.js"); const eligibleCount = postEpisodeFindings.length > 0 ? filterEligibleFindings(postEpisodeFindings).length : 0; return { /* ── CU synthesis result (mirrors production synthesizeFn) ─── */ currentUnderstanding, /* ── updatedSituationGraph (NEW — for longitudinal Episode N+1 state feeding) ─── */ updatedSituationGraph: updatedGraph, /* ── postEpisodeContributions (NEW — accumulated Contributions for longitudinal tracking) ─── */ postEpisodeContributions: effectiveContributions, /* ── postEpisodeFindings (NEW — accumulated Findings for Episode N+1) ─── */ postEpisodeFindings: postEpisodeFindings, /* ── Accurate post-episode accumulated findings count ─── */ findingsCheckpoint: { total: postEpisodeFindings.length, eligible: eligibleCount, }, /* ── graph checkpoint (from production result) ─── */ graphCheckpoint, /* ── Timing ─── */ modelElapsedMs, endToEndElapsedMs, providerGenerateCalls, modelName: episodeResult.modelName ?? null, /* ── CU synthesis error diagnostic (NEW — safe structural shape on validation failure) ─── */ synthesisErrorDiagnostic, /* ── Answer meaning (from production result) ─── */ userSupportedMeaning: episodeResult?.proposal?.answerMeaning?.userSupportedMeaning ?? null, possibleInference: episodeResult?.proposal?.answerMeaning?.possibleInference ?? null, }; } module.exports = { runLiveExperiment, extractLiveExperimentCheckpoint, _extractReasoningState, composeDeepInvestigationCheckpoint };