/** * Canonical live-update experiment harness. * * Loads .env.local, validates Ollama configuration, invokes updateCase() * through the real production path, and captures the standard reasoning * checkpoints needed by reasoning experiments. * * Usage (from a test file): * const { runLiveExperiment } = require("./tests/graph/live-update-experiment-helper.cjs"); * * const result = await runLiveExperiment({ * graph: /* SituationGraph fixture *\/, * previousQuestion: "Is risk a hard constraint?", * answer: "Risk matters more to me.", * }); */ const dotenv = require("dotenv"); dotenv.config({ path: ".env.local" }); function assertRequiredEnv(name) { const value = process.env[name]; if (!value) { throw new Error( `Live experiment requires ${name}. Set it in .env.local.\n` + `Found: OLLAMA_BASE_URL=${process.env.OLLAMA_BASE_URL ?? "(missing)"}, ` + `OLLAMA_MODEL=${process.env.OLLAMA_MODEL ?? "(missing)"}` ); } return value; } /** * Run one live experiment through the real updateCase() production path. * * @param {object} params * @param {object} params.graph — SituationGraph fixture (must match situationGraphSchema) * @param {string} params.previousQuestion * @param {string} params.answer * @returns {Promise} Checkpoints: * - userSupportedMeaning — extracted from answer meaning * - possibleInference — extracted from answer meaning * - rawAnswerCategory — deterministic category derived from raw answer text * - proposedMeaningCategory — deterministic category derived from userSupportedMeaning * - proposalValidation — { success, errors } * - compatibilityGuard — { passed, warnings, violations } * - graphMutation — { nodes: [...], edges: [...] } | null * - selectedQuestion — { id?, question? } | null * - behaviourSelection — string | null * - reasoningState — { turnCount, health, phase, progress, unknownStatuses } */ async function runLiveExperiment({ graph, previousQuestion, answer }) { const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL"); const model = assertRequiredEnv("OLLAMA_MODEL"); if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") { throw new Error( `Live experiment harness refuses to use localhost fallback. ` + `OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.` ); } const { updateCase } = await import( "../../lib/graph/orchestrator.js" ); const result = await updateCase({ situationGraph: graph, previousQuestion, answer }); // Extract the standard checkpoints const answerMeaning = result?.proposal?.answerMeaning || {}; const rawAnswerCat = _deriveCategory(answer); const supportedCat = _deriveCategory(answerMeaning.userSupportedMeaning || ""); return { userSupportedMeaning: answerMeaning.userSupportedMeaning ?? null, possibleInference: answerMeaning.possibleInference ?? null, rawAnswerCategory: rawAnswerCat, proposedMeaningCategory: supportedCat, proposalValidation: result?.proposalValidation ?? { success: false, errors: [] }, compatibilityGuard: result?.compatibilityApplied !== undefined ? { passed: result.compatibilityApplied, warnings: [], violations: [] } : { passed: false, warnings: [], violations: [] }, graphMutation: result?.appliedGraph ? { nodes: result.appliedGraph.nodes ?? null, edges: result.appliedGraph.edges ?? null } : null, selectedQuestion: result?.proposal?.selectedQuestion ?? null, behaviourSelection: result?.behaviourSelection ?? null, reasoningState: _extractReasoningState(result), }; } /** * Minimal deterministic category derivation from free-text (mirrors the production pipeline). */ function _deriveCategory(text) { if (!text || !text.trim()) return "none"; const lower = text.toLowerCase(); // Conditional patterns if (/normally\s+(?:avoid|skip|not\s+take|would\s+n't|can\'t)/i.test(lower) && /(might|could|would\s+(?:accept|allow|take|do))/i.test(lower)) { return "conditional_tradeoff"; } // Contrast / but patterns if (/but\s+i?\s*(don\'t|cannot|can\'t|won\'t|will\s+not)/i.test(lower)) { return "qualified_support"; } // Explicit hard constraint if (/(hard\s+constraint|must\s+(?:not|never|always)|can\'?\s*t(?:o)\s*(?:not|be\s+able\s+to)|absolutely\s+cannot)/i.test(lower)) { return "hard_constraint"; } // Strong preference / must positive if (/(must\s+(?:have|do|get)|absolutely\s+(?:need|require)|cannot\s+proceed\s+without)/i.test(lower)) { return "strong_preference"; } // Relative importance if (/more\s+to\s+me|matters\s+more|higher\s+priority|top\s*priority/i.test(lower)) { return "relative_importance"; } // Support / contraindicate if (/support(s)?\b|confirm(s)?\b|validates?\b/i.test(lower)) { return "supports_decision"; } if (/contradicts?\b|against\s+it\b|i\s*don\'?\s*t\s*(?:think\s+so|agree)\b/i.test(lower)) { return "contradicts_decision"; } return "cannot_determine"; } function _extractReasoningState(result) { const rs = result?.reasoningState; if (!rs) return null; return { turnCount: rs.turnCount ?? null, health: rs.health ?? null, phase: rs.phase ?? null, progress: rs.progress ?? null, unknownStatuses: (rs.unknownStatuses || []).map(s => s.id + ":" + s.status), }; } module.exports = { runLiveExperiment };