/** * Canonical live-update experiment harness. * * Loads .env.local, validates Ollama configuration, invokes updateCase() * through the real production path, and captures the standard reasoning * checkpoints needed by reasoning experiments. * * Usage (from a test file): * const { runLiveExperiment } = require("./tests/graph/live-update-experiment-helper.cjs"); * * const result = await runLiveExperiment({ * graph: /* SituationGraph fixture *\/, * previousQuestion: "Is risk a hard constraint?", * answer: "Risk matters more to me.", * }); */ const dotenv = require("dotenv"); dotenv.config({ path: ".env.local" }); function assertRequiredEnv(name) { const value = process.env[name]; if (!value) { throw new Error( `Live experiment requires ${name}. Set it in .env.local.\n` + `Found: OLLAMA_BASE_URL=${process.env.OLLAMA_BASE_URL ?? "(missing)"}, ` + `OLLAMA_MODEL=${process.env.OLLAMA_MODEL ?? "(missing)"}` ); } return value; } /** * Extract checkpoint data from a raw updateCase() result object. * * Pure function — no side effects, no I/O. Intended for use both by * `runLiveExperiment()` (apparatus wrapper) and by deterministic tests * that pass mock results directly. * * @param {object} result — raw result from updateCase() * @param {object} params — { graph?, answer? } * - graph: SituationGraph input fixture (for findingsCount total derived from input) * - answer: previous question text for deriveCategory fallback * @returns {object} checkpoint object per spec contract */ function extractLiveExperimentCheckpoint(result, params) { const answerMeaning = result?.proposal?.answerMeaning || {}; // Graph structural checkpoint (available from updatedSituationGraph only when present) const updatedGraph = result?.updatedSituationGraph; const graphCheckpoint = updatedGraph ? { nodeCount: updatedGraph.nodes?.length ?? 0, edgeCount: updatedGraph.edges?.length ?? 0, resolvedNodeIds: updatedGraph.resolvedNodeIds ?? [], } : null; return { // ── timing (available via wrapper) ──────────────────── modelElapsedMs: result?.modelElapsedMs !== undefined ? result.modelElapsedMs : null, endToEndElapsedMs: result?.endToEndElapsedMs !== undefined ? result.endToEndElapsedMs : null, providerGenerateCalls: result != null && result.providerGenerateCalls != null ? result.providerGenerateCalls : 0, modelName: result?.modelName || null, // ── answer meaning extraction ───────────────────────── userSupportedMeaning: answerMeaning.userSupportedMeaning ?? null, possibleInference: answerMeaning.possibleInference ?? null, rawAnswerCategory: _deriveCategory(params?.answer), proposedMeaningCategory: _deriveCategory(answerMeaning.userSupportedMeaning || ""), // ── validation / guard checkpoints ─────────────────── proposalValidation: result?.proposalValidation ?? { success: false, errors: [] }, compatibilityGuard: result?.compatibilityApplied !== undefined ? { passed: result.compatibilityApplied, warnings: [], violations: [] } : { passed: false, warnings: [], violations: [] }, // ── graph mutation checkpoint (from appliedGraph) ──── graphMutation: result?.appliedGraph ? { nodes: result.appliedGraph.nodes ?? null, edges: result.appliedGraph.edges ?? null } : null, // ── graph structural checkpoint (available from updatedSituationGraph) ─ graphCheckpoint, // ── findings checkpoint (from input graph fixture, not production path) ─ findingsCount: { total: (params?.graph && Array.isArray(params.graph)) ? params.graph.length : (params?.graph && params.graph.nodes != null) ? params.graph.nodes.length : 0, eligible: "UNAVAILABLE", // not canonically observable without production contract change }, // ── CU narrative (NOT captured — synthesis is a separate seam from updateCase()) ─ currentUnderstanding: "UNAVAILABLE", // ── candidate evidence (UNAVAILABLE without changing production code) ──────── eligibleFindingPropositions: "UNAVAILABLE", // ── helper internals (for diagnostic / test verification) ──────────────────── selectedQuestion: result?.proposal?.selectedQuestion ?? null, behaviourSelection: result?.behaviourSelection ?? null, reasoningState: _extractReasoningState(result), }; } /** * Minimal deterministic category derivation from free-text (mirrors the production pipeline). */ function _deriveCategory(text) { if (!text || !text.trim()) return "none"; const lower = text.toLowerCase(); // Conditional patterns if (/normally\s+(?:avoid|skip|not\s+take|would\s+n't|can\'t)/i.test(lower) && /(might|could|would\s+(?:accept|allow|take|do))/i.test(lower)) { return "conditional_tradeoff"; } // Contrast / but patterns if (/but\s+i?\s*(don\'t|cannot|can\'t|won\'t|will\s+not)/i.test(lower)) { return "qualified_support"; } // Explicit hard constraint if (/(hard\s+constraint|must\s+(?:not|never|always)|can\'?\s*t(?:o)\s*(?:not|be\s+able\s+to)|absolutely\s+cannot)/i.test(lower)) { return "hard_constraint"; } // Strong preference / must positive if (/(must\s+(?:have|do|get)|absolutely\s+(?:need|require)|cannot\s+proceed\s+without)/i.test(lower)) { return "strong_preference"; } // Relative importance if (/more\s+to\s+me|matters\s+more|higher\s+priority|top\s*priority/i.test(lower)) { return "relative_importance"; } // Support / contraindicate if (/support(s)?\b|confirm(s)?\b|validates?\b/i.test(lower)) { return "supports_decision"; } if (/contradicts?\b|against\s+it\b|i\s*don\'?\s*t\s*(?:think\s+so|agree)\b/i.test(lower)) { return "contradicts_decision"; } return "cannot_determine"; } /** * Extract reasoning state from a raw updateCase() result object. * Exported for deterministic test verification of preserved behaviour. */ function _extractReasoningState(result) { const rs = result?.reasoningState; if (!rs) return null; return { turnCount: rs.turnCount ?? null, health: rs.health ?? null, phase: rs.phase ?? null, progress: rs.progress ?? null, unknownStatuses: (rs.unknownStatuses || []).map(s => s.id + ":" + s.status), }; } /** * Run one live experiment through the real updateCase() production path. */ async function runLiveExperiment({ graph, previousQuestion, answer }) { const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL"); const model = assertRequiredEnv("OLLAMA_MODEL"); if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") { throw new Error( `Live experiment harness refuses to use localhost fallback. ` + `OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.` ); } const { updateCase } = await import( "../../lib/graph/orchestrator.js" ); const { getProvider } = await import("../../lib/llm/provider.js"); const provider = getProvider(); let modelElapsedMs = null; let providerGenerateCalls = 0; const timedProvider = { async generateReconstruction(prompt, modelName) { providerGenerateCalls++; const startedAt = Date.now(); const response = await provider.generateReconstruction(prompt, modelName); modelElapsedMs = Date.now() - startedAt; return response; }, }; const endToEndStartedAt = Date.now(); const result = await updateCase( { situationGraph: graph, previousQuestion, answer }, { provider: timedProvider }, ); const endToEndElapsedMs = Date.now() - endToEndStartedAt; return extractLiveExperimentCheckpoint(result, { graph, answer }); } module.exports = { runLiveExperiment, extractLiveExperimentCheckpoint, _extractReasoningState };