/** * Behaviour Selection Counterfactual — Experiment 41 * * Compare two small, passive alternatives for reducing Acknowledge dominance: * * Variant A — evaluate Summarise/Pause before Acknowledge (priority reordering); * Variant B — existing priority with Acknowledge exclusions (phase/progress/health gates). * * Production selector is NOT modified. Variants exist only in this test file. */ import { describe, it, expect } from "vitest"; import assessInvestigationState from "@/lib/assessment/investigation-state-assessor.js"; import selectBehaviour from "@/lib/behaviour-selection/behaviour-selector.js"; /* ═══════════════════════════════════════════════════ 1. Minimal assessment builder (not mutated) ═══════════════════════════════════════════════════ */ function mkAssessment(opts = {}) { return { version: "v0.1", assessedAt: new Date().toISOString(), confidence: opts.overallConfidence || "medium", phase: opts.phase ?? { value: "cannot_determine", confidence: "low", signals: [], evidence: { resolvedNodeCount: 0, activeUnknownCount: 0, unknownResolutionRatio: null, observationDensity: 0, evidenceDepth: "insufficient" } }, progress: opts.progress ?? { value: "cannot_determine", confidence: "low", signals: [], evidence: { turnCount: 0, recentResolutionsLastTurn: 0, newUnknownsPerTurn: null, repeatedNodeIds: [] } }, conversationHealth: opts.conversationHealth ?? { value: "cannot_determine", confidence: "low", signals: [], evidence: { questionTypeDistribution: null, activeUnknownCount: 0, resolvedNodeRatio: null, hasActiveQuestion: false, summaryLength: 0 } } }; } /* ═══════════════════════════════════════════════════ 2. Variant A — Specific behaviours before Acknowledge ═══════════════════════════════════════════════════ */ function selectSummariseV2(assessment) { if (assessment.phase.value === "synthesising") return { behaviour: "summarise", confidence: "high", reason: "Investigation is in synthesising phase — connected observations have accumulated and a restatement of current understanding will compress without losing detail." }; if (assessment.phase.value === "concluding") return { behaviour: "summarise", confidence: "high", reason: "Investigation is concluding — a summary of resolved understanding provides closure anchor before the user decides next steps." }; if (assessment.phase.evidence?.resolvedNodeCount >= 3 && assessment.progress.value === "steady") return { behaviour: "summarise", confidence: "medium", reason: "Three or more items resolved with steady progress — enough accumulated understanding warrants a compression pass." }; return null; } function selectPauseV2(assessment) { if (assessment.phase.value === "focusing" && assessment.progress.value === "stalled") return { behaviour: "pause", confidence: "high", reason: "Focusing phase with stalled progress — the investigation has reached a single active unknown but momentum has stopped. Hold space rather than pushing for more." }; if (assessment.conversationHealth.value === "user_overloaded") return { behaviour: "pause", confidence: "medium", reason: "User appears overloaded — reduce pressure by acknowledging progress before inviting further contribution." }; return null; } function selectAcknowledgeV2(assessment) { if (assessment.conversationHealth.value === "healthy" && assessment.phase.confidence !== "low") return { behaviour: "acknowledge", confidence: "medium", reason: "Healthy conversation with established context — user provided useful information that warrants acknowledgment before introducing new uncertainty." }; return null; } function selectClarifyV2(assessment) { if (assessment.conversationHealth.value === "too_broad") return { behaviour: "clarify", confidence: "high", reason: "Conversation health is too broad — investigation may be spreading too thin. Narrow focus through a specific clarification question." }; if (assessment.phase.value === "orienting" && assessment.phase.evidence?.observationDensity < 3) return { behaviour: "clarify", confidence: "medium", reason: "Investigation is in orienting phase with insufficient observations (< 3). A targeted clarification question will anchor the starting point." }; return null; } function selectVariantA(assessment) { if (!assessment || !assessment.conversationHealth || !assessment.phase) return { behaviour: "continue", confidence: "low", reason: "No assessment available — defaulting to continue." }; const summarise = selectSummariseV2(assessment); if (summarise) return { ...summarise, priority: 1 }; const pause = selectPauseV2(assessment); if (pause) return { ...pause, priority: 2 }; const clarify = selectClarifyV2(assessment); if (clarify) return { ...clarify, priority: 3 }; const acknowledge = selectAcknowledgeV2(assessment); if (acknowledge) return { ...acknowledge, priority: 4 }; return { behaviour: "continue", confidence: "low", reason: "No explicit rule matched — defaulting to continue." }; } /* ═══════════════════════════════════════════════════ 3. Variant B — Existing priority with Acknowledge exclusions ═══════════════════════════════════════════════════ */ function isAcknowledgeExcluded(assessment) { if (["synthesising", "concluding"].includes(assessment.phase.value)) return true; if (assessment.progress.value === "stalled") return true; if (assessment.conversationHealth.value === "user_overloaded") return true; return false; } function selectVariantB(assessment) { if (!assessment || !assessment.conversationHealth || !assessment.phase) return { behaviour: "continue", confidence: "low", reason: "No assessment available — defaulting to continue." }; const acknowledge = selectAcknowledgeV2(assessment); if (acknowledge && !isAcknowledgeExcluded(assessment)) return { ...acknowledge, priority: 1 }; const clarify = selectClarifyV2(assessment); if (clarify) return { ...clarify, priority: 2 }; const summarise = selectSummariseV2(assessment); if (summarise) return { ...summarise, priority: 3 }; const pause = selectPauseV2(assessment); if (pause) return { ...pause, priority: 4 }; return { behaviour: "continue", confidence: "low", reason: "No explicit rule matched — defaulting to continue." }; } /* ═══════════════════════════════════════════════════ 4. Real-scenario fixtures (imported from Exp 39/40) ═══════════════════════════════════════════════════ */ function mkN(id, label, opts = {}) { const kind = opts.kind || "unknown"; const status = opts.status || (kind === "unknown" ? "unknown" : "known"); const confidence = opts.confidence || (kind === "unknown" ? "low" : "high"); return { id, label, description: label, kind, status, confidence, evidenceIds: [], dependsOn: [], affects: [], childIds: [] }; } function buildInput(turn) { return { situationGraph: { centralStatement: turn.centralStatement, currentSummary: turn.currentSummary, nodes: turn.nodes, edges: [], activeUnknownNodeId: turn.activeUnknownNodeId, resolvedNodeIds: turn.resolvedNodeIds }, selectedQuestion: turn.selectedQuestion, noQuestionReason: turn.noQuestionReason, diagnostics: { promptVersion: "v0.4", modelName: "mock-ollama", responseDurationMs: 0, validationStatus: "valid", nodeCount: turn.nodes.length, edgeCount: 0, reasoningPattern: turn.diagnosticReasoningPattern || null } }; } function getRealTurns() { return [ { scenario: "long-investigation", turnNumber: 0, centralStatement: "Should we enter the European market with our SaaS analytics platform?", nodes: [mkN("obs-1", "Current revenue is $2M ARR in the US market only", { kind: "observation", status: "known", confidence: "high" }), mkN("state-1", "Evaluating European market entry", { kind: "state", status: "provisional", confidence: "medium" }), mkN("u-1", "Whether there is genuine demand for our category in Europe")], resolvedNodeIds: [], activeUnknownNodeId: "u-1", selectedQuestion: { nodeId: "u-1", question: "How large and mature is the analytics SaaS market in Europe?", reason: "market_validity" }, currentSummary: "We are US-based. The first question before any expansion is whether demand exists.", diagnosticReasoningPattern: "market_validity" }, { scenario: "long-investigation", turnNumber: 3, centralStatement: "Should we enter the European market with our SaaS analytics platform?", nodes: [mkN("obs-1", "Current revenue is $2M ARR in the US market only", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-2", "European analytics SaaS market valued at approximately €8B and growing 15% annually", { kind: "observation", status: "known", confidence: "medium" }), mkN("obs-3", "Our platform does not currently support EU data residency requirements", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-4", "Achieving compliance would require approximately 6 months and $500K engineering investment", { kind: "observation", status: "known", confidence: "medium" }), mkN("state-1", "Evaluating European market entry", { kind: "state", status: "provisional", confidence: "medium" }), mkN("u-1", "Whether there is genuine demand for our category in Europe", { status: "resolved", confidence: "medium" }), mkN("u-2", "Whether our product is suitable for European compliance requirements", { status: "resolved", confidence: "high" }), mkN("u-3", "Whether the cost of achieving compliance is justified by the market size", { status: "resolved", confidence: "medium" }), mkN("u-4", "Whether we have competitive differentiation against existing European players")], resolvedNodeIds: ["u-1", "u-2", "u-3"], activeUnknownNodeId: "u-4", selectedQuestion: { nodeId: "u-4", question: "What differentiates our platform against established European competitors?", reason: "competitive_analysis" }, currentSummary: "Compliance is feasible. The remaining question is competitive edge.", diagnosticReasoningPattern: "competitive_analysis" }, { scenario: "long-investigation", turnNumber: 4, centralStatement: "Should we enter the European market with our SaaS analytics platform?", nodes: [mkN("obs-1", "Current revenue is $2M ARR in the US market only", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-2", "European analytics SaaS market valued at approximately €8B and growing 15% annually", { kind: "observation", status: "known", confidence: "medium" }), mkN("obs-3", "Our platform does not currently support EU data residency requirements", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-4", "Achieving compliance would require approximately 6 months and $500K engineering investment", { kind: "observation", status: "known", confidence: "medium" }), mkN("obs-5", "Our real-time collaboration feature has no direct European equivalent", { kind: "observation", status: "provisional", confidence: "medium" }), mkN("state-1", "Evaluating European market entry", { kind: "state", status: "provisional", confidence: "medium" }), mkN("u-1", "Whether there is genuine demand for our category in Europe", { status: "resolved", confidence: "medium" }), mkN("u-2", "Whether our product is suitable for European compliance requirements", { status: "resolved", confidence: "high" }), mkN("u-3", "Whether the cost of achieving compliance is justified by the market size", { status: "resolved", confidence: "medium" }), mkN("u-4", "Whether we have competitive differentiation against existing European players", { status: "resolved", confidence: "medium" })], resolvedNodeIds: ["u-1", "u-2", "u-3", "u-4"], activeUnknownNodeId: null, selectedQuestion: null, noQuestionReason: "All investigation areas resolved.", currentSummary: "European market entry is justified if compliance is achieved and the real-time collaboration feature is positioned as differentiator.", diagnosticReasoningPattern: null }, { scenario: "contradictory-evidence", turnNumber: 0, centralStatement: "Two consultants give opposite recommendations on which supplier to choose for a $2M procurement.", nodes: [mkN("obs-1", "Consultant A recommends Supplier X: lower cost, proven track record", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-2", "Consultant B recommends Supplier Y: better integration capability, higher risk but long-term upside", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-3", "Supplier X has 15+ years in the sector; Supplier Y has 2 years and mixed client reviews", { kind: "observation", status: "known", confidence: "medium" }), mkN("obs-4", "Our current infrastructure is compatible with neither supplier out of the box", { kind: "observation", status: "known", confidence: "high" }), mkN("state-1", "Evaluating $2M procurement against conflicting expert advice", { kind: "state", status: "provisional", confidence: "medium" }), mkN("u-1", "Whether the conflict is genuine or reflects different evaluation criteria")], resolvedNodeIds: [], activeUnknownNodeId: "u-1", selectedQuestion: { nodeId: "u-1", question: "Are the consultants evaluating the same criteria, or are they measuring different things?", reason: "comparability_check" }, currentSummary: "Conflicting recommendations from two experts. The first uncertainty is whether we are comparing the same dimensions.", diagnosticReasoningPattern: "comparability_check" }, { scenario: "contradictory-evidence", turnNumber: 1, centralStatement: "Two consultants give opposite recommendations on which supplier to choose for a $2M procurement.", nodes: [mkN("obs-1", "Consultant A recommends Supplier X: lower cost, proven track record", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-2", "Consultant B recommends Supplier Y: better integration capability, higher risk but long-term upside", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-3", "Supplier X has 15+ years in the sector; Supplier Y has 2 years and mixed client reviews", { kind: "observation", status: "known", confidence: "medium" }), mkN("obs-4", "Our current infrastructure is compatible with neither supplier out of the box", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-5", "The consultants used different evaluation weights: cost 60% vs integration 60%", { kind: "observation", status: "known", confidence: "medium" }), mkN("state-1", "Evaluating $2M procurement against conflicting expert advice", { kind: "state", status: "provisional", confidence: "medium" }), mkN("u-1", "Whether the conflict is genuine or reflects different evaluation criteria", { status: "resolved", confidence: "high" }), mkN("u-2", "Which supplier's strengths align with our strategic priorities")], resolvedNodeIds: ["u-1"], activeUnknownNodeId: "u-2", selectedQuestion: { nodeId: "u-2", question: "Does cost or integration capability matter more to the organisation over a 3-year horizon?", reason: "evidence_quality" }, currentSummary: "The conflict reflects different evaluation weights. The next uncertainty is strategic alignment.", diagnosticReasoningPattern: "evidence_quality" }, { scenario: "contradictory-evidence", turnNumber: 2, centralStatement: "Two consultants give opposite recommendations on which supplier to choose for a $2M procurement.", nodes: [mkN("obs-1", "Consultant A recommends Supplier X: lower cost, proven track record", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-2", "Consultant B recommends Supplier Y: better integration capability, higher risk but long-term upside", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-3", "Supplier X has 15+ years in the sector; Supplier Y has 2 years and mixed client reviews", { kind: "observation", status: "known", confidence: "medium" }), mkN("obs-4", "Our current infrastructure is compatible with neither supplier out of the box", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-5", "The consultants used different evaluation weights: cost 60% vs integration 60%", { kind: "observation", status: "known", confidence: "medium" }), mkN("obs-6", "Our strategic plan prioritises long-term capability over short-term cost savings", { kind: "observation", status: "known", confidence: "high" }), mkN("state-1", "Evaluating $2M procurement against conflicting expert advice", { kind: "state", status: "provisional", confidence: "medium" }), mkN("u-1", "Whether the conflict is genuine or reflects different evaluation criteria", { status: "resolved", confidence: "high" }), mkN("u-2", "Which supplier's strengths align with our strategic priorities", { status: "resolved", confidence: "medium" }), mkN("u-3", "Whether the integration risk of Supplier Y is manageable with internal resources")], resolvedNodeIds: ["u-1", "u-2"], activeUnknownNodeId: "u-3", selectedQuestion: { nodeId: "u-3", question: "Do we have the internal capacity to manage Supplier Y's integration risk?", reason: "alternative_explanation" }, currentSummary: "Strategic priorities favour integration capability. The remaining uncertainty is operational feasibility.", diagnosticReasoningPattern: "alternative_explanation" }, { scenario: "short-early", turnNumber: 0, centralStatement: "A manufacturing company reports complaints increased by 35% while production increased by 40%.", nodes: [mkN("obs-1", "Complaints increased by 35%", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-2", "Production increased by 40%", { kind: "observation", status: "known", confidence: "high" }), mkN("state-1", "Current situation", { kind: "state", status: "provisional", confidence: "medium" }), mkN("u-1", "Whether the two figures cover the same period")], resolvedNodeIds: [], activeUnknownNodeId: "u-1", selectedQuestion: { nodeId: "u-1", question: "Were the complaint and production figures measured over the same period?", reason: "comparability_check" }, currentSummary: "Two changes have been reported, but we do not yet know whether the figures are directly comparable.", diagnosticReasoningPattern: "comparability_check" } ]; } /* ═══════════════════════════════════════════════════ 5. Evaluation helper ═══════════════════════════════════════════════════ */ function evaluateTurn(turn) { const input = buildInput(turn); const assessment = assessInvestigationState(input); // Deep-clone to verify immutability later const originalHealthValue = assessment.conversationHealth.value; const originalPhaseConfidence = assessment.phase.confidence; const originalProgressValue = assessment.progress.value; const existingSel = selectBehaviour(assessment); const varASel = selectVariantA(assessment); const varBSel = selectVariantB(assessment); // Verify immutability expect(assessment.conversationHealth.value).toBe(originalHealthValue); expect(assessment.phase.confidence).toBe(originalPhaseConfidence); expect(assessment.progress.value).toBe(originalProgressValue); return { scenario: turn.scenario, turnNumber: turn.turnNumber, centralStatementShort: turn.centralStatement.substring(0, 55) + (turn.centralStatement.length > 55 ? "…" : ""), phase: assessment.phase.value, phaseConfidence: assessment.phase.confidence, progress: assessment.progress.value, health: assessment.conversationHealth.value, existingSelection: existingSel.behaviour, variantASelection: varASel.behaviour, variantBSelection: varBSel.behaviour, _assessment: assessment }; } /* ═══════════════════════════════════════════════════ 6. Classification helper ═══════════════════════════════════════════════════ */ function classifyChange(turnIdx, originalSel, newSel) { if (originalSel === newSel) return "unchanged"; const turn = getRealTurns()[turnIdx]; const input = buildInput(turn); const assessment = assessInvestigationState(input); const { phase, progress, health } = { phase: assessment.phase.value, progress: assessment.progress.value, health: assessment.conversationHealth.value }; if (newSel === "summarise" && ["synthesising", "concluding"].includes(phase)) return "sensible"; if (newSel === "summarise" && phase.evidence?.resolvedNodeCount >= 3 && progress === "steady") return "sensible"; if (newSel === "pause" && ((phase === "focusing" && progress === "stalled") || health === "user_overloaded")) return "sensible"; if (newSel === "acknowledge" && health === "healthy" && assessment.phase.confidence !== "low") return "sensible"; // Check whether original was sensible and new one is questionable const origWasSensible = (originalSel === "acknowledge" && health === "healthy" && assessment.phase.confidence !== "low"); if (origWasSensible && newSel !== "acknowledge") { return "questionable"; // Original was sensible, the variant changed it to something less fitting } if (newSel === "continue") return "sensible"; if (newSel === "summarise" || newSel === "pause") return "questionable"; // no clear rule match found return "cannot determine"; } /* ═══════════════════════════════════════════════════ Test suite — Experiment 41 ═══════════════════════════════════════════════════ */ describe("Experiment 41 — Counterfactual: Acknowledge Priority Alternatives", () => { /* ── Variant A contract tests ──────────────── */ describe("Variant A implements only priority reordering", () => { it("prioritises Summarise before Acknowledge", () => { const result = selectVariantA(mkAssessment({ phase: { value: "concluding", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } })); expect(result.behaviour).toBe("summarise"); }); it("prioritises Pause before Acknowledge when conditions match", () => { const result = selectVariantA(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "stalled",confidence: "high" }, conversationHealth: { value: "healthy", confidence: "high" } })); expect(result.behaviour).toBe("pause"); }); it("does not change non-conflicting selections", () => { const result = selectVariantA(mkAssessment({ phase: { value: "exploring", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_broad", confidence: "high" } })); expect(result.behaviour).toBe("clarify"); // no ack conditions met, clarify is only match }); it("preserves Continue fallback", () => { const result = selectVariantA(mkAssessment({ phase: { value: "exploring", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } })); expect(result.behaviour).toBe("continue"); }); it("preserves Clarify rule when health=too_broad", () => { const result = selectVariantA(mkAssessment({ phase: { value: "orienting", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_broad", confidence: "high" } })); expect(result.behaviour).toBe("clarify"); }); }); /* ── Variant B contract tests ──────────────── */ describe("Variant B implements only Acknowledge exclusions", () => { it("blocks Acknowledge when phase=synthesising", () => { const result = selectVariantB(mkAssessment({ phase: { value: "synthesising", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } })); expect(result.behaviour).toBe("summarise"); // Acknowledge excluded, summarise fires next }); it("blocks Acknowledge when phase=concluding", () => { const result = selectVariantB(mkAssessment({ phase: { value: "concluding", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } })); expect(result.behaviour).toBe("summarise"); // Acknowledge excluded, summarise fires next }); it("blocks Acknowledge when progress=stalled", () => { const result = selectVariantB(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "stalled", confidence: "high" }, conversationHealth: { value: "healthy", confidence: "medium" } })); expect(result.behaviour).toBe("pause"); // Acknowledge excluded, pause fires next }); it("blocks Acknowledge when health=user_overloaded", () => { const result = selectVariantB(mkAssessment({ phase: { value: "exploring", confidence: "medium" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "user_overloaded", confidence: "medium" } })); expect(result.behaviour).toBe("pause"); // Acknowledge excluded, pause fires next }); it("preserves Acknowledge when no exclusion conditions match", () => { const result = selectVariantB(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } })); expect(result.behaviour).toBe("acknowledge"); // focusing≠synthesising/concluding, progress=steady≠stalled, health=healthy≠user_overloaded }); it("preserves Continue fallback", () => { const result = selectVariantB(mkAssessment({ phase: { value: "exploring", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } })); expect(result.behaviour).toBe("continue"); }); it("preserves Clarify rule when health=too_broad and phase low confidence", () => { const result = selectVariantB(mkAssessment({ phase: { value: "orienting", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_broad", confidence: "high" } })); expect(result.behaviour).toBe("clarify"); }); it("preserves Summarise rule when phase=synthesising and Acknowledge excluded", () => { const result = selectVariantB(mkAssessment({ phase: { value: "synthesising", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } })); expect(result.behaviour).toBe("summarise"); // Acknowledge excluded, summarise fires at priority 3 }); }); /* ── Key scenario reachability tests ───────── */ describe("Key scenario reachability", () => { it("concluding state can reach Summarise under Variant A", () => { const result = selectVariantA(mkAssessment({ phase: { value: "concluding", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } })); expect(result.behaviour).toBe("summarise"); }); it("stalled focusing state can reach Pause under Variant A", () => { const result = selectVariantA(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "stalled", confidence: "high" }, conversationHealth: { value: "healthy", confidence: "high" } })); expect(result.behaviour).toBe("pause"); }); it("healthy ordinary progress can still reach Acknowledge under Variant A", () => { const result = selectVariantA(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } })); expect(result.behaviour).toBe("acknowledge"); // summarise/pause not eligible, ack is 4th but wins }); it("concluding state can reach Summarise under Variant B", () => { const result = selectVariantB(mkAssessment({ phase: { value: "concluding", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } })); expect(result.behaviour).toBe("summarise"); }); it("stalled focusing state can reach Pause under Variant B", () => { const result = selectVariantB(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "stalled", confidence: "high" }, conversationHealth: { value: "healthy", confidence: "medium" } })); expect(result.behaviour).toBe("pause"); }); it("healthy ordinary progress can still reach Acknowledge under Variant B", () => { const result = selectVariantB(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } })); expect(result.behaviour).toBe("acknowledge"); // no exclusion conditions match (focusing≠synthesising/concluding, steady≠stalled) }); it("fallback remains Continue when no rule matches", () => { const resultA = selectVariantA(mkAssessment({ phase: { value: "exploring", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } })); const resultB = selectVariantB(mkAssessment({ phase: { value: "exploring", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } })); expect(resultA.behaviour).toBe("continue"); expect(resultB.behaviour).toBe("continue"); }); it("early low-confidence state falls back safely under both variants", () => { const result = selectVariantA(mkAssessment({ phase: { value: "cannot_determine", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } })); expect(result.behaviour).toBe("continue"); const resultB = selectVariantB(mkAssessment({ phase: { value: "cannot_determine", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } })); expect(resultB.behaviour).toBe("continue"); }); }); /* ── Existing selector unchanged ───────────── */ describe("Existing selector results remain unchanged", () => { it("returns acknowledge for healthy phase with confidence (base behaviour)", () => { const result = selectBehaviour(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } })); expect(result.behaviour).toBe("acknowledge"); }); it("returns continue for null input (base behaviour)", () => { const result = selectBehaviour(null); expect(result.behaviour).toBe("continue"); }); it("only outputs valid behaviours (base contract)", () => { const behaviours = []; for (const phase of ["cannot_determine", "exploring", "orienting", "deepening", "focusing", "synthesising", "concluding"]) { for (const conf of ["high", "medium", "low"]) { const result = selectBehaviour(mkAssessment({ phase: { value: phase, confidence: conf }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "cannot_determine", confidence: "low" } })); behaviours.push(result.behaviour); } } for (const b of behaviours) expect(["acknowledge", "clarify", "summarise", "pause", "continue"]).toContain(b); }); }); /* ── Seven real assessment turns comparison ─ */ describe("Seven real assessment turns compared across all selectors", () => { let results; beforeEach(() => { const turns = getRealTurns(); results = turns.map(t => evaluateTurn(t)); }); it("evaluates all seven turns", () => expect(results).toHaveLength(7)); it("outputs are deterministic across calls", () => { for (let i = 0; i < results.length; i++) { const turn2 = getRealTurns()[i]; const result2 = evaluateTurn(turn2); expect(result2.existingSelection).toBe(results[i].existingSelection); expect(result2.variantASelection).toBe(results[i].variantASelection); expect(result2.variantBSelection).toBe(results[i].variantBSelection); } }); it("inputs are not mutated", () => { const turns = getRealTurns(); for (const turn of turns) { const input = buildInput(turn); const snapshot = JSON.stringify(input); assessInvestigationState(input); selectVariantA(assessInvestigationState(JSON.parse(snapshot))); selectVariantB(assessInvestigationState(JSON.parse(snapshot))); expect(JSON.stringify(input)).toBe(snapshot); } }); it("reports assessor output for each turn", () => { for (const r of results) { console.log(`\n=== ${r.scenario} turn ${r.turnNumber} ===`); console.log(` phase=${r.phase}(conf:${r.phaseConfidence}), progress=${r.progress}, health=${r.health}`); console.log(` existing: ${r.existingSelection} | variant A: ${r.variantASelection} | variant B: ${r.variantBSelection}`); } }); it("records whether each change appears sensible/questionable", () => { const classifications = {}; for (let i = 0; i < results.length; i++) { const r = results[i]; classifications[`existing-${r.scenario}-t${r.turnNumber}`] = "unchanged"; // baseline if (r.existingSelection !== r.variantASelection) { classifications[`variantA-${r.scenario}-t${r.turnNumber}`] = classifyChange(i, r.existingSelection, r.variantASelection); } else { classifications[`variantA-${r.scenario}-t${r.turnNumber}`] = "unchanged"; } if (r.existingSelection !== r.variantBSelection) { classifications[`variantB-${r.scenario}-t${r.turnNumber}`] = classifyChange(i, r.existingSelection, r.variantBSelection); } else { classifications[`variantB-${r.scenario}-t${r.turnNumber}`] = "unchanged"; } } console.log("\n=== Selection Classifications ==="); for (const [key, val] of Object.entries(classifications)) { console.log(` ${key}: ${val}`); } global._exp41_classifications = classifications; }); it("turn-level summary", () => { const distA = { acknowledge: 0, clarify: 0, summarise: 0, pause: 0, continue: 0 }; const distB = { acknowledge: 0, clarify: 0, summarise: 0, pause: 0, continue: 0 }; for (const r of results) { distA[r.variantASelection]++; distB[r.variantBSelection]++; } console.log("\n=== Existing Distribution ==="); const existingDist = { acknowledge: 0, clarify: 0, summarise: 0, pause: 0, continue: 0 }; for (const r of results) existingDist[r.existingSelection]++; for (const [beh, count] of Object.entries(existingDist)) console.log(` ${beh}: ${count} (${((count/7)*100).toFixed(0)}%)`); console.log("\n=== Variant A Distribution ==="); for (const [beh, count] of Object.entries(distA)) console.log(` ${beh}: ${count} (${((count/7)*100).toFixed(0)}%)`); console.log("\n=== Variant B Distribution ==="); for (const [beh, count] of Object.entries(distB)) console.log(` ${beh}: ${count} (${((count/7)*100).toFixed(0)}%)`); global._exp41_distA = distA; global._exp41_distB = distB; }); }); /* ── Synthetic safety checks ───────────────── */ describe("Synthetic safety checks", () => { it("healthy mid-investigation progress can reach Acknowledge under both variants", () => { const a = selectVariantA(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } })); const b = selectVariantB(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } })); expect(a.behaviour).toBe("acknowledge"); expect(b.behaviour).toBe("acknowledge"); }); it("concluding state reaches Summarise under both variants", () => { const a = selectVariantA(mkAssessment({ phase: { value: "concluding", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } })); const b = selectVariantB(mkAssessment({ phase: { value: "concluding", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } })); expect(a.behaviour).toBe("summarise"); expect(b.behaviour).toBe("summarise"); }); it("focusing + stalled reaches Pause under both variants", () => { const a = selectVariantA(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "stalled", confidence: "high" }, conversationHealth: { value: "healthy", confidence: "medium" } })); const b = selectVariantB(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "stalled", confidence: "high" }, conversationHealth: { value: "healthy", confidence: "medium" } })); expect(a.behaviour).toBe("pause"); expect(b.behaviour).toBe("pause"); }); it("user_overloaded reaches Pause under both variants", () => { const a = selectVariantA(mkAssessment({ phase: { value: "exploring", confidence: "medium" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "user_overloaded", confidence: "medium" } })); const b = selectVariantB(mkAssessment({ phase: { value: "exploring", confidence: "medium" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "user_overloaded", confidence: "medium" } })); expect(a.behaviour).toBe("pause"); expect(b.behaviour).toBe("pause"); }); it("early low-confidence state falls back to Continue under both variants", () => { const a = selectVariantA(mkAssessment({ phase: { value: "exploring", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } })); const b = selectVariantB(mkAssessment({ phase: { value: "exploring", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } })); expect(a.behaviour).toBe("continue"); expect(b.behaviour).toBe("continue"); }); it("state where no explicit rule matches returns Continue under both variants", () => { const a = selectVariantA(mkAssessment({ phase: { value: "deepening", confidence: "medium" }, progress: { value: "steady", confidence: "medium" } })); const b = selectVariantB(mkAssessment({ phase: { value: "deepening", confidence: "medium" }, progress: { value: "steady", confidence: "medium" } })); expect(a.behaviour).toBe("continue"); expect(b.behaviour).toBe("continue"); }); it("neither variant causes Pause too early in normal progress", () => { const a = selectVariantA(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } })); const b = selectVariantB(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } })); expect(a.behaviour).not.toBe("pause"); expect(b.behaviour).not.toBe("pause"); }); it("neither variant causes Summarise too early in normal progress", () => { const a = selectVariantA(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } })); expect(a.behaviour).not.toBe("summarise"); }); it("early incomplete states fall back safely under both variants", () => { const a = selectVariantA(mkAssessment({ phase: { value: "cannot_determine", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } })); const b = selectVariantB(mkAssessment({ phase: { value: "cannot_determine", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } })); expect(a.behaviour).toBe("continue"); expect(b.behaviour).toBe("continue"); }); }); /* ── Both variants produce identical results across tested turns ─ */ describe("Variant A vs Variant B — divergence analysis", () => { it("both variants converge on the same two genuine changes (concluding→summarise, stalled→pause) despite different mechanisms", () => { const turns = getRealTurns(); // Variant A diverges at long-investigation t3 (focusing with 3 resolved items) // because its resolvedNodeCount >= 3 rule fires without phase context — a side-effect. // Variant B correctly preserves Acknowledge there via explicit exclusion list. // Both converge on the two genuine changes: concluding→summarise and stalled→pause. const divergeIdx = 1; // long-investigation t3 (Variant A over-summarises here) expect(selectVariantA(assessInvestigationState(buildInput(turns[divergeIdx]))).behaviour).toBe("summarise"); expect(selectVariantB(assessInvestigationState(buildInput(turns[divergeIdx]))).behaviour).toBe("acknowledge"); // Both converge: concluding turn const a4 = selectVariantA(assessInvestigationState(buildInput(turns[2]))); const b4 = selectVariantB(assessInvestigationState(buildInput(turns[2]))); expect(a4.behaviour).toBe("summarise"); expect(b4.behaviour).toBe("summarise"); // Both converge: stalled focusing turn const c1 = selectVariantA(assessInvestigationState(buildInput(turns[4]))); const d1 = selectVariantB(assessInvestigationState(buildInput(turns[4]))); expect(c1.behaviour).toBe("pause"); expect(d1.behaviour).toBe("pause"); }); }); /* ── Specific expected outcomes ────────────── */ describe("Specific expected outcomes", () => { it("concluding turn (long-investigation t4) selects Summarise under both variants", () => { const turns = getRealTurns(); const r = evaluateTurn(turns[2]); // long-investigation t4 expect(r.variantASelection).toBe("summarise"); expect(r.variantBSelection).toBe("summarise"); }); it("stalled focusing turn (contradictory-evidence t1) selects Pause under both variants", () => { const turns = getRealTurns(); const r = evaluateTurn(turns[4]); // contradictory-evidence t1 expect(r.variantASelection).toBe("pause"); expect(r.variantBSelection).toBe("pause"); }); it("ordinary healthy progress still allows Acknowledge under Variant B (Variant A over-summarises mid-focus)", () => { const turns = getRealTurns(); // long-investigation t3: Variant A incorrectly produces summarise because resolvedNodeCount>=3 fires before acknowledging (side-effect) expect(evaluateTurn(turns[1]).variantASelection).toBe("summarise"); expect(evaluateTurn(turns[1]).variantBSelection).toBe("acknowledge"); // contradictory-t0 and t2: both preserve acknowledge for (const idx of [3, 5]) { const r = evaluateTurn(turns[idx]); expect(r.variantASelection).toBe("acknowledge"); expect(r.variantBSelection).toBe("acknowledge"); } }); it("early incomplete states fall back to Continue under both variants", () => { const turns = getRealTurns(); for (const idx of [0, 6]) { // long-investigation t0, short-early-t0 const r = evaluateTurn(turns[idx]); expect(r.variantASelection).toBe("continue"); expect(r.variantBSelection).toBe("continue"); } }); }); });