test: establish canonical live reasoning experiment harness

2026-08-09 11:56:10 +01:00
parent 4aa1492c8d
commit e6f784261b
3 changed files with 198 additions and 0 deletions
@@ -0,0 +1,149 @@
+/**
+ * Canonical live-update experiment harness.
+ *
+ * Loads .env.local, validates Ollama configuration, invokes updateCase()
+ * through the real production path, and captures the standard reasoning
+ * checkpoints needed by reasoning experiments.
+ *
+ * Usage (from a test file):
+ *   const { runLiveExperiment } = require("./tests/graph/live-update-experiment-helper.cjs");
+ *
+ *   const result = await runLiveExperiment({
+ *     graph: /* SituationGraph fixture *\/,
+ *     previousQuestion: "Is risk a hard constraint?",
+ *     answer: "Risk matters more to me.",
+ *   });
+ */
+
+const dotenv = require("dotenv");
+dotenv.config({ path: ".env.local" });
+
+function assertRequiredEnv(name) {
+  const value = process.env[name];
+  if (!value) {
+    throw new Error(
+      `Live experiment requires ${name}. Set it in .env.local.\n` +
+      `Found: OLLAMA_BASE_URL=${process.env.OLLAMA_BASE_URL ?? "(missing)"}, ` +
+      `OLLAMA_MODEL=${process.env.OLLAMA_MODEL ?? "(missing)"}`
+    );
+  }
+  return value;
+}
+
+/**
+ * Run one live experiment through the real updateCase() production path.
+ *
+ * @param {object} params
+ * @param {object} params.graph       — SituationGraph fixture (must match situationGraphSchema)
+ * @param {string} params.previousQuestion
+ * @param {string} params.answer
+ * @returns {Promise<object>} Checkpoints:
+ *   - userSupportedMeaning    — extracted from answer meaning
+ *   - possibleInference       — extracted from answer meaning
+ *   - rawAnswerCategory       — deterministic category derived from raw answer text
+ *   - proposedMeaningCategory — deterministic category derived from userSupportedMeaning
+ *   - proposalValidation      — { success, errors }
+ *   - compatibilityGuard      — { passed, warnings, violations }
+ *   - graphMutation           — { nodes: [...], edges: [...] } | null
+ *   - selectedQuestion        — { id?, question? } | null
+ *   - behaviourSelection      — string | null
+ *   - reasoningState          — { turnCount, health, phase, progress, unknownStatuses }
+ */
+async function runLiveExperiment({ graph, previousQuestion, answer }) {
+  const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL");
+  const model   = assertRequiredEnv("OLLAMA_MODEL");
+
+  if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
+    throw new Error(
+      `Live experiment harness refuses to use localhost fallback. ` +
+      `OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.`
+    );
+  }
+
+  const { updateCase } = await import(
+    "../lib/graph/orchestrator.js"
+  );
+
+  const result = await updateCase({ situationGraph: graph, previousQuestion, answer });
+
+  // Extract the standard checkpoints
+  const answerMeaning  = result?.proposal?.answerMeaning || {};
+  const rawAnswerCat   = _deriveCategory(answer);
+  const supportedCat   = _deriveCategory(answerMeaning.userSupportedMeaning || "");
+
+  return {
+    userSupportedMeaning: answerMeaning.userSupportedMeaning ?? null,
+    possibleInference:    answerMeaning.possibleInference    ?? null,
+    rawAnswerCategory:    rawAnswerCat,
+    proposedMeaningCategory: supportedCat,
+    proposalValidation:   result?.proposalValidation ?? { success: false, errors: [] },
+    compatibilityGuard:   result?.compatibilityApplied !== undefined
+      ? { passed: result.compatibilityApplied, warnings: [], violations: [] }
+      : { passed: false, warnings: [], violations: [] },
+    graphMutation:        result?.appliedGraph
+      ? { nodes: result.appliedGraph.nodes ?? null, edges: result.appliedGraph.edges ?? null }
+      : null,
+    selectedQuestion:     result?.proposal?.selectedQuestion ?? null,
+    behaviourSelection:   result?.behaviourSelection ?? null,
+    reasoningState:       _extractReasoningState(result),
+  };
+}
+
+/**
+ * Minimal deterministic category derivation from free-text (mirrors the production pipeline).
+ */
+function _deriveCategory(text) {
+  if (!text || !text.trim()) return "none";
+
+  const lower = text.toLowerCase();
+
+  // Conditional patterns
+  if (/normally\s+(?:avoid|skip|not\s+take|would\s+n't|can\'t)/i.test(lower) &&
+      /(might|could|would\s+(?:accept|allow|take|do))/i.test(lower)) {
+    return "conditional_tradeoff";
+  }
+
+  // Contrast / but patterns
+  if (/but\s+i?\s*(don\'t|cannot|can\'t|won\'t|will\s+not)/i.test(lower)) {
+    return "qualified_support";
+  }
+
+  // Explicit hard constraint
+  if (/(hard\s+constraint|must\s+(?:not|never|always)|can\'?\s*t(?:o)\s*(?:not|be\s+able\s+to)|absolutely\s+cannot)/i.test(lower)) {
+    return "hard_constraint";
+  }
+
+  // Strong preference / must positive
+  if (/(must\s+(?:have|do|get)|absolutely\s+(?:need|require)|cannot\s+proceed\s+without)/i.test(lower)) {
+    return "strong_preference";
+  }
+
+  // Relative importance
+  if (/more\s+to\s+me|matters\s+more|higher\s+priority|top\s*priority/i.test(lower)) {
+    return "relative_importance";
+  }
+
+  // Support / contraindicate
+  if (/support(s)?\b|confirm(s)?\b|validates?\b/i.test(lower)) {
+    return "supports_decision";
+  }
+  if (/contradicts?\b|against\s+it\b|i\s*don\'?\s*t\s*(?:think\s+so|agree)\b/i.test(lower)) {
+    return "contradicts_decision";
+  }
+
+  return "cannot_determine";
+}
+
+function _extractReasoningState(result) {
+  const rs = result?.reasoningState;
+  if (!rs) return null;
+  return {
+    turnCount:     rs.turnCount      ?? null,
+    health:        rs.health          ?? null,
+    phase:         rs.phase           ?? null,
+    progress:      rs.progress        ?? null,
+    unknownStatuses: (rs.unknownStatuses || []).map(s => s.id + ":" + s.status),
+  };
+}
+
+module.exports = { runLiveExperiment };