diff --git a/CLAUDE.md b/CLAUDE.md index f3ae4c8..34ebb5a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,6 +28,13 @@ user experience. Do not assume an architectural redesign is wanted. +## Live experiment harness rule + +When running reasoning experiments, use the canonical harness at +`tests/graph/live-update-experiment-helper.cjs`. Never create a new harness, +enumerate `/api/tags`, probe localhost, or discover/substitute models during +normal reasoning experiments. + ## Standard validation For UI-only work, normally run: diff --git a/docs/task-context-packs.md b/docs/task-context-packs.md index bbd1d92..5f6a119 100644 --- a/docs/task-context-packs.md +++ b/docs/task-context-packs.md @@ -24,6 +24,48 @@ - the task requires an undocumented contract; - the experiment begins expanding into several capabilities. +### Live experiment execution route + +A canonical live-update harness exists at `tests/graph/live-update-experiment-helper.cjs`. +It loads `.env.local`, validates required variables, invokes the real `updateCase()` production +entry point, and returns standard reasoning checkpoints (userSupportedMeaning, possibleInference, +rawAnswerCategory, proposedMeaningCategory, proposalValidation, compatibilityGuard, graphMutation, +selectedQuestion, behaviourSelection, reasoningState). + +**Execution pattern:** + +```js +const { runLiveExperiment } = require("./tests/graph/live-update-experiment-helper.cjs"); + +const result = await runLiveExperiment({ + graph: /* SituationGraph fixture *\/, + previousQuestion: "Is risk a hard constraint?", + answer: "Risk matters more to me.", +}); + +// Checkpoints available on `result`: +// result.userSupportedMeaning +// result.possibleInference +// result.rawAnswerCategory +// result.proposedMeaningCategory +// result.proposalValidation +// result.compatibilityGuard +// result.graphMutation +// result.selectedQuestion +// result.behaviourSelection +// result.reasoningState +``` + +**Required environment (from `.env.local`):** +- `process.env.OLLAMA_BASE_URL` — must be a real host (no localhost fallback) +- `process.env.OLLAMA_MODEL` — model name (e.g. `qwen-claude:latest`) + +The harness fails clearly if either variable is missing or OLLAMA_BASE_URL points to localhost. +It makes exactly one live Ollama call per invocation unless the experiment explicitly specifies otherwise. + +**Rule:** During normal reasoning experiments, never create a bespoke harness, enumerate `/api/tags`, +probe localhost, or discover/substitute another model. Use the canonical harness above. + ## Pack 2 — UI and Mock Work ### Always read diff --git a/tests/graph/live-update-experiment-helper.cjs b/tests/graph/live-update-experiment-helper.cjs new file mode 100644 index 0000000..98aba4a --- /dev/null +++ b/tests/graph/live-update-experiment-helper.cjs @@ -0,0 +1,149 @@ +/** + * Canonical live-update experiment harness. + * + * Loads .env.local, validates Ollama configuration, invokes updateCase() + * through the real production path, and captures the standard reasoning + * checkpoints needed by reasoning experiments. + * + * Usage (from a test file): + * const { runLiveExperiment } = require("./tests/graph/live-update-experiment-helper.cjs"); + * + * const result = await runLiveExperiment({ + * graph: /* SituationGraph fixture *\/, + * previousQuestion: "Is risk a hard constraint?", + * answer: "Risk matters more to me.", + * }); + */ + +const dotenv = require("dotenv"); +dotenv.config({ path: ".env.local" }); + +function assertRequiredEnv(name) { + const value = process.env[name]; + if (!value) { + throw new Error( + `Live experiment requires ${name}. Set it in .env.local.\n` + + `Found: OLLAMA_BASE_URL=${process.env.OLLAMA_BASE_URL ?? "(missing)"}, ` + + `OLLAMA_MODEL=${process.env.OLLAMA_MODEL ?? "(missing)"}` + ); + } + return value; +} + +/** + * Run one live experiment through the real updateCase() production path. + * + * @param {object} params + * @param {object} params.graph — SituationGraph fixture (must match situationGraphSchema) + * @param {string} params.previousQuestion + * @param {string} params.answer + * @returns {Promise} Checkpoints: + * - userSupportedMeaning — extracted from answer meaning + * - possibleInference — extracted from answer meaning + * - rawAnswerCategory — deterministic category derived from raw answer text + * - proposedMeaningCategory — deterministic category derived from userSupportedMeaning + * - proposalValidation — { success, errors } + * - compatibilityGuard — { passed, warnings, violations } + * - graphMutation — { nodes: [...], edges: [...] } | null + * - selectedQuestion — { id?, question? } | null + * - behaviourSelection — string | null + * - reasoningState — { turnCount, health, phase, progress, unknownStatuses } + */ +async function runLiveExperiment({ graph, previousQuestion, answer }) { + const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL"); + const model = assertRequiredEnv("OLLAMA_MODEL"); + + if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") { + throw new Error( + `Live experiment harness refuses to use localhost fallback. ` + + `OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.` + ); + } + + const { updateCase } = await import( + "../lib/graph/orchestrator.js" + ); + + const result = await updateCase({ situationGraph: graph, previousQuestion, answer }); + + // Extract the standard checkpoints + const answerMeaning = result?.proposal?.answerMeaning || {}; + const rawAnswerCat = _deriveCategory(answer); + const supportedCat = _deriveCategory(answerMeaning.userSupportedMeaning || ""); + + return { + userSupportedMeaning: answerMeaning.userSupportedMeaning ?? null, + possibleInference: answerMeaning.possibleInference ?? null, + rawAnswerCategory: rawAnswerCat, + proposedMeaningCategory: supportedCat, + proposalValidation: result?.proposalValidation ?? { success: false, errors: [] }, + compatibilityGuard: result?.compatibilityApplied !== undefined + ? { passed: result.compatibilityApplied, warnings: [], violations: [] } + : { passed: false, warnings: [], violations: [] }, + graphMutation: result?.appliedGraph + ? { nodes: result.appliedGraph.nodes ?? null, edges: result.appliedGraph.edges ?? null } + : null, + selectedQuestion: result?.proposal?.selectedQuestion ?? null, + behaviourSelection: result?.behaviourSelection ?? null, + reasoningState: _extractReasoningState(result), + }; +} + +/** + * Minimal deterministic category derivation from free-text (mirrors the production pipeline). + */ +function _deriveCategory(text) { + if (!text || !text.trim()) return "none"; + + const lower = text.toLowerCase(); + + // Conditional patterns + if (/normally\s+(?:avoid|skip|not\s+take|would\s+n't|can\'t)/i.test(lower) && + /(might|could|would\s+(?:accept|allow|take|do))/i.test(lower)) { + return "conditional_tradeoff"; + } + + // Contrast / but patterns + if (/but\s+i?\s*(don\'t|cannot|can\'t|won\'t|will\s+not)/i.test(lower)) { + return "qualified_support"; + } + + // Explicit hard constraint + if (/(hard\s+constraint|must\s+(?:not|never|always)|can\'?\s*t(?:o)\s*(?:not|be\s+able\s+to)|absolutely\s+cannot)/i.test(lower)) { + return "hard_constraint"; + } + + // Strong preference / must positive + if (/(must\s+(?:have|do|get)|absolutely\s+(?:need|require)|cannot\s+proceed\s+without)/i.test(lower)) { + return "strong_preference"; + } + + // Relative importance + if (/more\s+to\s+me|matters\s+more|higher\s+priority|top\s*priority/i.test(lower)) { + return "relative_importance"; + } + + // Support / contraindicate + if (/support(s)?\b|confirm(s)?\b|validates?\b/i.test(lower)) { + return "supports_decision"; + } + if (/contradicts?\b|against\s+it\b|i\s*don\'?\s*t\s*(?:think\s+so|agree)\b/i.test(lower)) { + return "contradicts_decision"; + } + + return "cannot_determine"; +} + +function _extractReasoningState(result) { + const rs = result?.reasoningState; + if (!rs) return null; + return { + turnCount: rs.turnCount ?? null, + health: rs.health ?? null, + phase: rs.phase ?? null, + progress: rs.progress ?? null, + unknownStatuses: (rs.unknownStatuses || []).map(s => s.id + ":" + s.status), + }; +} + +module.exports = { runLiveExperiment };