diff --git a/tests/graph/live-update-experiment-helper-checkpoint.test.cjs b/tests/graph/live-update-experiment-helper-checkpoint.test.cjs new file mode 100644 index 0000000..417b095 --- /dev/null +++ b/tests/graph/live-update-experiment-helper-checkpoint.test.cjs @@ -0,0 +1,228 @@ +/** + * Deterministic checkpoint extraction tests for live-update-experiment-helper.cjs. + * + * Exercises the actual exported extractLiveExperimentCheckpoint seam — not a + * duplicated simulation. All assertions verify observable behaviour of the + * canonical extraction path. + * + * Tests: + * 1. CU narrative field is "UNAVAILABLE" (NOT captured - synthesis is a separate seam) + * 2. Structural counts are captured deterministically from graph state + * 3. Timing fields pass through unchanged + * 4. Missing optional evidence fields are "UNAVAILABLE" (not fabricated) + * 5. Existing answer/category/validation checkpoint behaviour preserved + * 6. Absent updated graph produces null structural checkpoint + */ + +const { extractLiveExperimentCheckpoint, _extractReasoningState } = require("./live-update-experiment-helper.cjs"); + +// ---- deriveCategory kept in test because it is NOT exported from the helper ---- +// (It is part of the answer-meaning extraction infrastructure, not the checkpoint seam) +function deriveCategory(text) { + if (!text || !text.trim()) return "none"; + const lower = text.toLowerCase(); + if (/normally\s+(?:avoid|skip|not\s+take|would\s+n't|can\'t)/i.test(lower) && + /(might|could|would\s+(?:accept|allow|take|do))/i.test(lower)) return "conditional_tradeoff"; + if (/but\s+i?\s*(don\'t|cannot|can\'t|won\'t|will\s+not)/i.test(lower)) return "qualified_support"; + if (/(hard\s+constraint|must\s+(?:not|never|always)|can\'?\s*t(?:o)\s*(?:not|be\s+able\s+to)|absolutely\s+cannot)/i.test(lower)) return "hard_constraint"; + if (/(must\s+(?:have|do|get)|absolutely\s+(?:need|require)|cannot\s+proceed\s+without)/i.test(lower)) return "strong_preference"; + if (/more\s+to\s+me|matters\s+more|higher\s+priority|top\s*priority/i.test(lower)) return "relative_importance"; + if (/support(s)?\b|confirm(s)?\b|validates?\b/i.test(lower)) return "supports_decision"; + if (/contradicts?\b|against\s+it\b|i\s*don\'?\s*t\s*(?:think\s+so|agree)\b/i.test(lower)) return "contradicts_decision"; + return "cannot_determine"; +} + +function makeGraphFixture(nodes, edges) { + return { nodes: nodes || [], edges: edges || [], resolvedNodeIds: [] }; +} + +function successResult(mElapsed, e2e) { + var nodes = [ + { id: "a", kind: "unknown", label: "A", description: "Node A", status: "known" }, + { id: "b", kind: "unknown", label: "B", description: "Node B", status: "supported" }, + { id: "c", kind: "unknown", label: "C", description: "Node C", status: "unknown" }, + ]; + var edges = [ + { fromNodeId: "a", toNodeId: "b", relationship: "supports" }, + { fromNodeId: "b", toNodeId: "c", relationship: "depends_on" }, + ]; + return { + success: true, + stage: "update_applied", + updatedSituationGraph: makeGraphFixture(nodes, edges), + appliedGraph: null, + proposal: { + answerMeaning: { + userSupportedMeaning: "User confirms supporting evidence matters", + possibleInference: null, + supportCategory: "supports_decision", + resolutionGuidance: null, + }, + selectedQuestion: { id: "q1", question: "Next question?" }, + }, + proposalValidation: { success: true, errors: [] }, + compatibilityApplied: true, + behaviourSelection: "deterministic-choice", + reasoningState: { + turnCount: 2, + health: "healthy", + phase: "deep-investigation", + progress: "resolving unknowns", + unknownStatuses: [{ id: "c", status: "unknown" }], + }, + modelElapsedMs: mElapsed, + endToEndElapsedMs: e2e, + providerGenerateCalls: 1, + }; +} + +function failureResult() { + return { + success: false, + stage: "proposal_validation", + proposal: null, + proposalValidation: { success: false, errors: ["invalid proposal"] }, + compatibilityApplied: undefined, + behaviourSelection: null, + reasoningState: null, + modelElapsedMs: null, + endToEndElapsedMs: null, + providerGenerateCalls: 0, + updatedSituationGraph: null, + appliedGraph: null, + }; +} + +// ---- assertion helpers ---- +var failures = 0; +function assert(cond, msg) { + if (!cond) { console.error("FAIL:", msg); failures++; } + else { console.log("PASS:", msg); } +} +function eq(a, e, msg) { + var aS = JSON.stringify(a); + var eS = JSON.stringify(e); + if (aS !== eS) { console.error("FAIL [" + msg + "]: expected " + eS + ", got " + aS); failures++; } + else { console.log("PASS [" + msg + "]: values match"); } +} + +// ---- SUCCESS PATH TESTS ---- +console.log("\n=== CHECKPOINT EXTRACTION - SUCCESS PATH ===\n"); + +var successParams = { + graph: makeGraphFixture([ + { id: "a", kind: "unknown", label: "A", description: "Node A", status: "known" }, + { id: "b", kind: "unknown", label: "B", description: "Node B", status: "supported" }, + ], [ + { fromNodeId: "a", toNodeId: "b", relationship: "supports" }, + ]), + previousQuestion: "Is risk a hard constraint?", + answer: "Risk matters more to me.", +}; + +var checkpoint = extractLiveExperimentCheckpoint(successResult(123, 456), successParams); + +// 1. CU text is "UNAVAILABLE" (not captured - separate seam) +assert(checkpoint.currentUnderstanding === "UNAVAILABLE", "CU narrative is UNAVAILABLE (not captured)"); + +// 2. Structural counts match updated graph fixture +eq(checkpoint.graphCheckpoint.nodeCount, 3, "nodeCount matches updated graph nodes length"); +eq(checkpoint.graphCheckpoint.edgeCount, 2, "edgeCount matches updated graph edges length"); +assert(Array.isArray(checkpoint.graphCheckpoint.resolvedNodeIds), "resolvedNodeIds is array"); + +// 3. Timing fields present and correct type +assert(typeof checkpoint.modelElapsedMs === "number", "modelElapsedMs is number"); +eq(checkpoint.modelElapsedMs, 123, "modelElapsedMs value matches provider timing"); +assert(typeof checkpoint.endToEndElapsedMs === "number", "endToEndElapsedMs is number"); +eq(checkpoint.endToEndElapsedMs, 456, "endToEndElapsedMs value matches wrapper timing"); + +// 4. Findings count from input graph (not production path) +eq(checkpoint.findingsCount.total, 2, "findings total matches input graph length"); +assert(checkpoint.findingsCount.eligible === "UNAVAILABLE", "eligible finding count is UNAVAILABLE"); + +// 5. CU evidence propositions UNAVAILABLE +assert(checkpoint.eligibleFindingPropositions === "UNAVAILABLE", "eligibleFindingPropositions is UNAVAILABLE"); + +// 6. Answer meaning extraction preserved +eq(checkpoint.userSupportedMeaning, "User confirms supporting evidence matters", "userSupportedMeaning extracted correctly"); +eq(checkpoint.possibleInference, null, "possibleInference is null when absent"); +eq(checkpoint.rawAnswerCategory, "relative_importance", "raw answer category derived from 'matters more'"); + +// 7. Proposal validation preserved +assert(checkpoint.proposalValidation.success === true, "proposalValidation.success is true on success"); +eq(Object.keys(checkpoint.proposalValidation).length, 2, "proposalValidation has {success, errors} keys"); + +// 8. Compatibility guard captured correctly +eq(checkpoint.compatibilityGuard.passed, true, "compatibilityGuard.passed reflects production result"); +eq(checkpoint.compatibilityGuard.warnings.length, 0, "compatibilityGuard warnings empty"); +eq(checkpoint.compatibilityGuard.violations.length, 0, "compatibilityGuard violations empty"); + +// 9. Graph mutation null when appliedGraph absent (existing behaviour) +assert(checkpoint.graphMutation === null, "graphMutation is null when appliedGraph absent"); + +// 10. Reasoning state preserved +assert(checkpoint.reasoningState !== null, "reasoningState present on success"); +eq(checkpoint.reasoningState.turnCount, 2, "reasoningState turnCount preserved"); +eq(checkpoint.reasoningState.health, "healthy", "reasoningState health preserved"); +eq(checkpoint.reasoningState.phase, "deep-investigation", "reasoningState phase preserved"); +assert(Array.isArray(checkpoint.reasoningState.unknownStatuses), "unknownStatuses is array"); +eq(checkpoint.reasoningState.unknownStatuses.length, 1, "one unknown status tracked"); + +// 11. selectedQuestion and behaviourSelection captured +assert(checkpoint.selectedQuestion !== null, "selectedQuestion captured"); +eq(checkpoint.selectedQuestion.question, "Next question?", "selectedQuestion text correct"); +eq(checkpoint.behaviourSelection, "deterministic-choice", "behaviourSelection captured correctly"); + +// 12. Provider call tracking +eq(checkpoint.providerGenerateCalls, 1, "providerGenerateCalls incremented once"); + +// ---- ABSENT GRAPH PATH TESTS (spec item #6) ---- +console.log("\n=== CHECKPOINT EXTRACTION - ABSENT UPDATED GRAPH ===\n"); + +// Use a result that has no updatedSituationGraph to test null structural checkpoint +var absentResult = { + success: true, + stage: "proposal_validated", + updatedSituationGraph: null, + appliedGraph: null, + proposal: { answerMeaning: { userSupportedMeaning: "", possibleInference: null }, selectedQuestion: null }, + proposalValidation: { success: true, errors: [] }, + compatibilityApplied: false, + behaviourSelection: null, + reasoningState: null, + modelElapsedMs: 200, + endToEndElapsedMs: 800, + providerGenerateCalls: 1, +}; + +var absentGraphParams = { + graph: null, + previousQuestion: "Is risk a hard constraint?", + answer: "Risk matters more to me.", +}; + +var absentCheckpoint = extractLiveExperimentCheckpoint(absentResult, absentGraphParams); + +// Structural checkpoint is null when updatedSituationGraph absent +assert(absentCheckpoint.graphCheckpoint === null, "graphCheckpoint is null when updated graph absent"); + +// CU still unavailable without production contract change +assert(absentCheckpoint.currentUnderstanding === "UNAVAILABLE", "CU remains UNAVAILABLE with absent graph"); + +// Evidence propositions still unavailable +assert(absentCheckpoint.eligibleFindingPropositions === "UNAVAILABLE", "eligble props still UNAVAILABLE"); + +// Timing fields still pass through +eq(absentCheckpoint.modelElapsedMs, 200, "modelElapsedMs passes through in absent-graph path"); +eq(absentCheckpoint.endToEndElapsedMs, 800, "endToEndElapsedMs passes through in absent-graph path"); + +// Reasoning state is null when reasoningState absent in result +assert(absentCheckpoint.reasoningState === null, "reasoningState is null when absent in result"); + +// Test _extractReasoningState directly with null input +var nullRs = _extractReasoningState(null); +assert(nullRs === null, "_extractReasoningState returns null for null input"); + +// ---- exit with failure count ---- +console.log("\n--- Total failures: " + failures + " ---\n"); +process.exit(failures > 0 ? 1 : 0); \ No newline at end of file diff --git a/tests/graph/live-update-experiment-helper.cjs b/tests/graph/live-update-experiment-helper.cjs index 1aa8168..3347681 100644 --- a/tests/graph/live-update-experiment-helper.cjs +++ b/tests/graph/live-update-experiment-helper.cjs @@ -31,85 +31,78 @@ function assertRequiredEnv(name) { } /** - * Run one live experiment through the real updateCase() production path. + * Extract checkpoint data from a raw updateCase() result object. * - * @param {object} params - * @param {object} params.graph — SituationGraph fixture (must match situationGraphSchema) - * @param {string} params.previousQuestion - * @param {string} params.answer - * @returns {Promise} Checkpoints: - * - userSupportedMeaning — extracted from answer meaning - * - possibleInference — extracted from answer meaning - * - rawAnswerCategory — deterministic category derived from raw answer text - * - proposedMeaningCategory — deterministic category derived from userSupportedMeaning - * - proposalValidation — { success, errors } - * - compatibilityGuard — { passed, warnings, violations } - * - graphMutation — { nodes: [...], edges: [...] } | null - * - selectedQuestion — { id?, question? } | null - * - behaviourSelection — string | null - * - reasoningState — { turnCount, health, phase, progress, unknownStatuses } + * Pure function — no side effects, no I/O. Intended for use both by + * `runLiveExperiment()` (apparatus wrapper) and by deterministic tests + * that pass mock results directly. + * + * @param {object} result — raw result from updateCase() + * @param {object} params — { graph?, answer? } + * - graph: SituationGraph input fixture (for findingsCount total derived from input) + * - answer: previous question text for deriveCategory fallback + * @returns {object} checkpoint object per spec contract */ -async function runLiveExperiment({ graph, previousQuestion, answer }) { - const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL"); - const model = assertRequiredEnv("OLLAMA_MODEL"); +function extractLiveExperimentCheckpoint(result, params) { + const answerMeaning = result?.proposal?.answerMeaning || {}; - if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") { - throw new Error( - `Live experiment harness refuses to use localhost fallback. ` + - `OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.` - ); - } - - const { updateCase } = await import( - "../../lib/graph/orchestrator.js" - ); - - const { getProvider } = await import("../../lib/llm/provider.js"); - const provider = getProvider(); - let modelElapsedMs = null; - let providerGenerateCalls = 0; - - const timedProvider = { - async generateReconstruction(prompt, modelName) { - providerGenerateCalls++; - const startedAt = Date.now(); - const response = await provider.generateReconstruction(prompt, modelName); - modelElapsedMs = Date.now() - startedAt; - return response; - }, - }; - - const endToEndStartedAt = Date.now(); - const result = await updateCase( - { situationGraph: graph, previousQuestion, answer }, - { provider: timedProvider }, - ); - const endToEndElapsedMs = Date.now() - endToEndStartedAt; - - // Extract the standard checkpoints - const answerMeaning = result?.proposal?.answerMeaning || {}; - const rawAnswerCat = _deriveCategory(answer); - const supportedCat = _deriveCategory(answerMeaning.userSupportedMeaning || ""); + // Graph structural checkpoint (available from updatedSituationGraph only when present) + const updatedGraph = result?.updatedSituationGraph; + const graphCheckpoint = updatedGraph + ? { + nodeCount: updatedGraph.nodes?.length ?? 0, + edgeCount: updatedGraph.edges?.length ?? 0, + resolvedNodeIds: updatedGraph.resolvedNodeIds ?? [], + } + : null; return { - modelElapsedMs, - endToEndElapsedMs, - providerGenerateCalls, - modelName: model, + // ── timing (available via wrapper) ──────────────────── + modelElapsedMs: result?.modelElapsedMs !== undefined ? result.modelElapsedMs : null, + endToEndElapsedMs: result?.endToEndElapsedMs !== undefined ? result.endToEndElapsedMs : null, + providerGenerateCalls: result != null && result.providerGenerateCalls != null ? result.providerGenerateCalls : 0, + modelName: result?.modelName || null, + + // ── answer meaning extraction ───────────────────────── userSupportedMeaning: answerMeaning.userSupportedMeaning ?? null, possibleInference: answerMeaning.possibleInference ?? null, - rawAnswerCategory: rawAnswerCat, - proposedMeaningCategory: supportedCat, - proposalValidation: result?.proposalValidation ?? { success: false, errors: [] }, - compatibilityGuard: result?.compatibilityApplied !== undefined + rawAnswerCategory: _deriveCategory(params?.answer), + proposedMeaningCategory: _deriveCategory(answerMeaning.userSupportedMeaning || ""), + + // ── validation / guard checkpoints ─────────────────── + proposalValidation: result?.proposalValidation ?? { success: false, errors: [] }, + compatibilityGuard: result?.compatibilityApplied !== undefined ? { passed: result.compatibilityApplied, warnings: [], violations: [] } : { passed: false, warnings: [], violations: [] }, - graphMutation: result?.appliedGraph + + // ── graph mutation checkpoint (from appliedGraph) ──── + graphMutation: result?.appliedGraph ? { nodes: result.appliedGraph.nodes ?? null, edges: result.appliedGraph.edges ?? null } : null, - selectedQuestion: result?.proposal?.selectedQuestion ?? null, - behaviourSelection: result?.behaviourSelection ?? null, - reasoningState: _extractReasoningState(result), + + // ── graph structural checkpoint (available from updatedSituationGraph) ─ + graphCheckpoint, + + // ── findings checkpoint (from input graph fixture, not production path) ─ + findingsCount: { + total: (params?.graph && Array.isArray(params.graph)) + ? params.graph.length + : (params?.graph && params.graph.nodes != null) + ? params.graph.nodes.length + : 0, + eligible: "UNAVAILABLE", // not canonically observable without production contract change + }, + + // ── CU narrative (NOT captured — synthesis is a separate seam from updateCase()) ─ + currentUnderstanding: "UNAVAILABLE", + + // ── candidate evidence (UNAVAILABLE without changing production code) ──────── + eligibleFindingPropositions: "UNAVAILABLE", + + // ── helper internals (for diagnostic / test verification) ──────────────────── + selectedQuestion: result?.proposal?.selectedQuestion ?? null, + behaviourSelection: result?.behaviourSelection ?? null, + reasoningState: _extractReasoningState(result), }; } @@ -158,6 +151,10 @@ function _deriveCategory(text) { return "cannot_determine"; } +/** + * Extract reasoning state from a raw updateCase() result object. + * Exported for deterministic test verification of preserved behaviour. + */ function _extractReasoningState(result) { const rs = result?.reasoningState; if (!rs) return null; @@ -170,4 +167,47 @@ function _extractReasoningState(result) { }; } -module.exports = { runLiveExperiment }; +/** + * Run one live experiment through the real updateCase() production path. + */ +async function runLiveExperiment({ graph, previousQuestion, answer }) { + const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL"); + const model = assertRequiredEnv("OLLAMA_MODEL"); + + if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") { + throw new Error( + `Live experiment harness refuses to use localhost fallback. ` + + `OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.` + ); + } + + const { updateCase } = await import( + "../../lib/graph/orchestrator.js" + ); + + const { getProvider } = await import("../../lib/llm/provider.js"); + const provider = getProvider(); + let modelElapsedMs = null; + let providerGenerateCalls = 0; + + const timedProvider = { + async generateReconstruction(prompt, modelName) { + providerGenerateCalls++; + const startedAt = Date.now(); + const response = await provider.generateReconstruction(prompt, modelName); + modelElapsedMs = Date.now() - startedAt; + return response; + }, + }; + + const endToEndStartedAt = Date.now(); + const result = await updateCase( + { situationGraph: graph, previousQuestion, answer }, + { provider: timedProvider }, + ); + const endToEndElapsedMs = Date.now() - endToEndStartedAt; + + return extractLiveExperimentCheckpoint(result, { graph, answer }); +} + +module.exports = { runLiveExperiment, extractLiveExperimentCheckpoint, _extractReasoningState };