From da3d35f437c2f20fa66862c0c78d9849ec29f0ed Mon Sep 17 00:00:00 2001 From: robbond Date: Thu, 6 Aug 2026 16:30:10 +0100 Subject: [PATCH] experiment: validate behaviour selection against real assessments --- docs/current-handoff.md | 22 +- docs/current-project-state.md | 8 +- docs/design-evolution-log.md | 60 +++ ...ehaviour-selection.real-assessment.test.js | 487 ++++++++++++++++++ 4 files changed, 563 insertions(+), 14 deletions(-) create mode 100644 tests/behaviour-selection.real-assessment.test.js diff --git a/docs/current-handoff.md b/docs/current-handoff.md index 26f3565..6a2307e 100644 --- a/docs/current-handoff.md +++ b/docs/current-handoff.md @@ -4,18 +4,18 @@ ## 1. Where We Left It -- Engine experiments are paused after Experiment 25B; -- UI experiments are paused; +- Engine experiments resumed with a passive validation; +- UI experiments remain paused; - Knowledge-management experiments are complete; -- Experiment 38 validated that a cold session recovers project state from three documents alone; -- The working tree was clean at commit `544573a` (Exp 37). +- Experiment 39 tested the existing Behaviour Selection module against real Investigation State Assessment outputs across three scenarios; +- Acknowledge dominates (71% of selections) because it fires first when health=healthy, blocking Summarise/Pause/Clarify even in concluding or stalled states. > This handoff describes the latest stopping point only. When work moves on, replace stale current-work details rather than appending another historical note. Historical experiment and commit information belongs in `docs/design-evolution-log.md`. ## 2. What Is True Now - Main active engine path: deterministic reasoning pipeline (scenario reconstruction, graph update, unknown selection, question formulation, turn orchestration). -- Passive experimental classifiers from Experiments 18–25B remain isolated diagnostic layers; none control the user-facing investigation. +- Passive experimental classifiers from Experiments 18–25B remain isolated diagnostic layers; none control the user-facing investigation. Behaviour Selection was passively evaluated against real assessment outputs in Experiment 39 — it produced all valid behaviours but with skewed distribution (Acknowledge 71%). - Keyword and phrase-based scope detection remains provisional scaffolding. - `docs/current-project-state.md` is the main entry point for active project state. - `docs/task-context-packs.md` chooses the minimum context documents for each work type. @@ -34,9 +34,13 @@ Experiment 38 tested whether a genuinely cold session (no prior conversation con **Commit:** pending (experiment: validate cold-start project recovery) — to be committed this session. +Experiment 39 resumed reasoning experiments with a passive validation of Behaviour Selection against real Investigation State Assessment outputs. Seven turns across three scenarios were evaluated. Acknowledge dominated (71%) because it fires at priority 1 whenever health=healthy, even in terminal and stalled states where Summarise or Pause would be more useful. The assessor→selector contract aligns cleanly; no transformation is needed between pipeline stages. All five behaviours remain reachable but some never appear in typical scenarios (Clarify requires too_broad health which few fixtures produce). Status pending Rob's review. + ## 5. What Remains Open -- Whether tasks crossing pack boundaries can still stay concise without loading excessive context (not tested by this experiment). +- Whether the Acknowledge→Summarise priority conflict can be resolved without breaking existing rule conditions; +- Whether behaviour selection produces different patterns in investigation domains other than supplier procurement, market entry, and product comparison; +- Which rules (Acknowledge, Clarify, Summarise, Pause) fire most frequently across a larger set of scenarios. ### When This Knowledge-Management Phase Is Complete @@ -82,8 +86,8 @@ Answer before continuing: --- -*Created by Experiment 34. Updated by Experiment 38. Branch: `feature/user-workspace-ux-v0.7`.* +*Created by Experiment 34. Updated by Experiments 38, 39. Branch: `feature/user-workspace-ux-v0.7`.* -### Return-to-Work Note (Experiment 38) +### Return-to-Work Note (Experiment 39) -Experiment 38 was performed from a cold session with no prior conversation context and confirmed that the reduced context system works as designed. A genuinely fresh session recovered the complete project state—active capabilities, passive diagnostic layers, paused work items, and context-pack routing—from just three documents without reading the full design-evolution log or any archived material. All seven knowledge-management completion criteria are confirmed met. The handoff itself was found to be accurate with no stale or ambiguous statements; one structural update was made (resolving the open item about handoff longevity). Engine and UI experiments remain paused pending Rob's review. Branch: `feature/user-workspace-ux-v0.7`. First file to inspect when returning: `docs/current-handoff.md`, then `docs/current-project-state.md` and the Engine Experiment pack per the routing table. The next action is discussion, not automatic implementation. +Experiment 39 resumed reasoning experiments with a passive validation of Behaviour Selection against real Investigation State Assessment outputs across three scenarios (long investigation, contradictory evidence, short early). No user-facing engine behaviour changed. The selector produced all valid behaviours but with skewed distribution: Acknowledge dominated at 71% because it fires first when health=healthy, even in terminal/concluding states where Summarise would be more useful — this is an acknowledged priority conflict, not a contract mismatch. The assessor→selector pipeline aligns cleanly. Clarify and Pause never fired (no too_broad health produced; other rules blocked). Status pending Rob's review. Branch: `feature/user-workspace-ux-v0.7`. First file to inspect when returning: `docs/current-handoff.md`, then `docs/design-evolution-log.md` entry for Experiment 39, and `tests/behaviour-selection.real-assessment.test.js` for evaluation details. The next action is discussion about whether to refine the selector or move on. diff --git a/docs/current-project-state.md b/docs/current-project-state.md index 604d6a9..96566ec 100644 --- a/docs/current-project-state.md +++ b/docs/current-project-state.md @@ -46,7 +46,7 @@ These are what currently affect the working engine: The following were built during Experiments 18–25B. They are isolated diagnostic layers with no active integration into the user-facing investigation: - Investigation-state assessment (phase and progress classification); -- Behaviour selection from assessed state; +- Behaviour selection from assessed state — passively evaluated in Experiment 39 against real assessment outputs; produces all valid behaviours but Acknowledge dominates when health=healthy; - Decision condition status evaluation; - Question-to-condition relevance scoring; - Evidence direction classification (support, contradict, inform); @@ -75,7 +75,7 @@ The following were built during Experiments 18–25B. They are isolated diagnost ## 6. Work Currently Paused -- Engine experiments are paused after Experiment 25B; +- Engine experiments are at Experiment 39 (passive Behaviour Selection validation, awaiting review); - UI experiments are paused; - Knowledge-management experiments are complete (confirmed by Experiment 38 cold-start validation); - Nothing historical has been deleted or archived yet. @@ -100,12 +100,10 @@ Historical documents are retained under `docs/archive/` and should be opened onl ## 8. Return-to-Work Summary -Engine experiments are paused after Experiment 25B, which established scope-aware condition status — distinguishing direct evidence from relevant-but-different claims by checking subject, timeframe and claim type. The passive classifier layers (Exps 18–25B) remain isolated; none yet control the user-facing investigation. Current focus is knowledge management: creating a short current-state entry point so future sessions do not need to read the full experiment history. +Engine experiments advanced to Experiment 39, a passive validation of Behaviour Selection against real Investigation State Assessment outputs. The existing selector produces all valid behaviours but Acknowledge dominates (71%) because it fires first when health=healthy. No user-facing engine behaviour changed. Knowledge-management phase is complete: all seven criteria met. No backlog items deleted or promoted. First document to read: `docs/current-project-state.md`. Then consult `.claude/architecture-guardrails.md` before any code changes and `docs/project-knowledge-inventory.md` for task-specific references. The full experiment history remains available in `docs/design-evolution-log.md` but is no longer default reading. -Engine experiments remain paused after Experiment 25B. UI experiments are paused. Knowledge-management phase is complete: all seven criteria met. No backlog items deleted or promoted. **First file to inspect when resuming:** `docs/current-project-state.md`. - ## Verification Marker Implementation status last checked against source: Experiment 28. diff --git a/docs/design-evolution-log.md b/docs/design-evolution-log.md index 3692e7c..349545c 100644 --- a/docs/design-evolution-log.md +++ b/docs/design-evolution-log.md @@ -2264,3 +2264,63 @@ All seven knowledge-management completion criteria are confirmed met by this col The cold-start validation passed. A genuinely fresh session understood the project state, chose the correct context pack, verified the resume boundary, produced a valid engine-work resume brief, and found no handoff defects — all from three documents alone. No source code was read or changed. The reduced context system works for sessions that did not help create the documents. Engine and UI experiments remain paused pending Rob's review. + +--- + +## Experiment 39 — Validate Behaviour Selection Against Real Assessment Outputs (2026-08-06) + +**Branch:** feature/user-workspace-ux-v0.7 + +### Hypothesis +The existing deterministic selector produces a useful rhythm across genuine assessment outputs without changing the active engine. If it repeatedly chooses one behaviour, chooses behaviours at the wrong time, or depends on signals the assessor does not actually produce, the experiment should expose that honestly. + +### Scenarios Evaluated (from `tests/investigation-state-assessor.test.js` fixture set) +1. **Long investigation** (3 turns: early → deepening → complete terminal) +2. **Contradictory evidence** (3 turns: two conflicting consultants, 0→1→2 resolved unknowns) +3. **Short early** (1 turn: two observations, first unknown, no resolution) + +### Behaviour Distribution (7 turns total) +- Acknowledge: 5 (71%) +- Continue: 2 (29%) +- Clarify: 0 (0%) +- Summarise: 0 (0%) +- Pause: 0 (0%) + +### Behaviour Sequence by Scenario +**Long investigation:** continue → acknowledge → acknowledge +- Turn 0: phase=cannot_determine, progress=cannot_determine, health=too_narrow → continue (no rule matched) +- Turn 3: phase=focusing, progress=steady, health=healthy → acknowledge +- Turn 4: phase=concluding, progress=steady, health=healthy → acknowledge + +**Contradictory evidence:** acknowledge → acknowledge → acknowledge +- Turn 0: phase=focusing, progress=cannot_determine, health=healthy → acknowledge +- Turn 1: phase=focusing, progress=stalled, health=healthy → acknowledge +- Turn 2: phase=focusing, progress=steady, health=healthy → acknowledge + +**Short early:** continue +- Turn 0: phase=exploring, progress=cannot_determine, health=healthy → continue + +### Sensible Selections (7 of 7) +All selections were classified as sensible per the selection's stated conditions. Acknowledge fires because `health=healthy AND phase confidence≠low` across most states. Continue fires when no specific rule matches (early/cannot_determine/exploring phases). + +### Questionable or Inappropriate Selections +**One notable pattern:** Summarise and Pause never fire, even in a concluding terminal state. This is not because the assessor fails to detect "concluding" — it does. It is because Acknowledge (priority 1) fires first when health=healthy, blocking Summarise (priority 3) from ever reaching its turn. This is an **acknowledgement/summarise priority conflict**: acknowledging a conclusion ("you've figured this out!") is not wrong, but "give me a summary" is more useful at terminal states. The current rule ordering does not distinguish "early healthy" from "concluding healthy." + +Clarify never fires because no test scenario produces `health=too_broad` — the assessor's "too_broad" trigger (activeUnknownCount > 3 AND resolved < 2) requires more nodes than any scenario in the fixture set has at that stage. + +Pause never fires because `health=user_overloaded` is never reached, and while contradictory-turn-1 has phase=focusing + progress=stalled, Acknowledge still blocks it. + +### Contract Alignment +Assessor → Selector contract aligns cleanly. The assessor produces all three dimensions (phase, progress, conversationHealth) with the fields the selector expects. No transformation needed between pipeline stages. + +### Whether Selector Appears Useful Enough for Another Passive Experiment +The existing selector works but its **behaviour variation is severely constrained by Acknowledge's priority position**. A next passive experiment should test whether reordering or refining the acknowledge condition (e.g., excluding concluding/terminal phases) produces more context-appropriate behaviour — without changing the assessor. + +### Status +Pending Rob's review. Five behaviours are too narrow for this to be definitive, and only three scenarios were tested. The dominant pattern (acknowledge in healthy states) may change with different investigation domains. + +### Documents Updated +- `docs/design-evolution-log.md` — this entry +- `docs/current-handoff.md` — return-to-work note replaced + +--- diff --git a/tests/behaviour-selection.real-assessment.test.js b/tests/behaviour-selection.real-assessment.test.js new file mode 100644 index 0000000..3f2c5bf --- /dev/null +++ b/tests/behaviour-selection.real-assessment.test.js @@ -0,0 +1,487 @@ +import { describe, it, expect } from "vitest"; +import assessInvestigationState from "@/lib/assessment/investigation-state-assessor.js"; +import selectBehaviour, { BEHAVIOUR_OPTIONS } from "@/lib/behaviour-selection/behaviour-selector.js"; + +/* ── Helper: build scenario fixture data inline ─────────── */ + +function mkN(id, label, opts = {}) { + const kind = opts.kind || "unknown"; + const status = opts.status || (kind === "unknown" ? "unknown" : "known"); + const confidence = opts.confidence || (kind === "unknown" ? "low" : "high"); + return { + id, label, description: label, kind, status, confidence, + evidenceIds: [], dependsOn: [], affects: [], childIds: [] + }; +} + +/* ── Scenario fixtures used in this experiment ─────────── */ + +function getEvaluationScenarios() { + return { + // SCENARIO A: Long / developing investigation (4-turn arc) + "long-investigation": [ + { // Turn 0 — early, single observation + centralStatement: "Should we enter the European market with our SaaS analytics platform?", + nodes: [ + mkN("obs-1", "Current revenue is $2M ARR in the US market only", { kind: "observation", status: "known", confidence: "high" }), + mkN("state-1", "Evaluating European market entry", { kind: "state", status: "provisional", confidence: "medium" }), + mkN("u-1", "Whether there is genuine demand for our category in Europe") + ], + resolvedNodeIds: [], activeUnknownNodeId: "u-1", + selectedQuestion: { nodeId: "u-1", question: "How large and mature is the analytics SaaS market in Europe?", reason: "market_validity" }, + currentSummary: "We are US-based. The first question before any expansion is whether demand exists.", + diagnosticReasoningPattern: "market_validity" + }, + { // Turn 3 — deepening, many resolved + centralStatement: "Should we enter the European market with our SaaS analytics platform?", + nodes: [ + mkN("obs-1", "Current revenue is $2M ARR in the US market only", { kind: "observation", status: "known", confidence: "high" }), + mkN("obs-2", "European analytics SaaS market valued at approximately €8B and growing 15% annually", { kind: "observation", status: "known", confidence: "medium" }), + mkN("obs-3", "Our platform does not currently support EU data residency requirements", { kind: "observation", status: "known", confidence: "high" }), + mkN("obs-4", "Achieving compliance would require approximately 6 months and $500K engineering investment", { kind: "observation", status: "known", confidence: "medium" }), + mkN("state-1", "Evaluating European market entry", { kind: "state", status: "provisional", confidence: "medium" }), + mkN("u-1", "Whether there is genuine demand for our category in Europe", { status: "resolved", confidence: "medium" }), + mkN("u-2", "Whether our product is suitable for European compliance requirements", { status: "resolved", confidence: "high" }), + mkN("u-3", "Whether the cost of achieving compliance is justified by the market size", { status: "resolved", confidence: "medium" }), + mkN("u-4", "Whether we have competitive differentiation against existing European players") + ], + resolvedNodeIds: ["u-1", "u-2", "u-3"], activeUnknownNodeId: "u-4", + selectedQuestion: { nodeId: "u-4", question: "What differentiates our platform against established European competitors?", reason: "competitive_analysis" }, + currentSummary: "Compliance is feasible. The remaining question is competitive edge.", + diagnosticReasoningPattern: "competitive_analysis" + }, + { // Turn 4 — complete, terminal + centralStatement: "Should we enter the European market with our SaaS analytics platform?", + nodes: [ + mkN("obs-1", "Current revenue is $2M ARR in the US market only", { kind: "observation", status: "known", confidence: "high" }), + mkN("obs-2", "European analytics SaaS market valued at approximately €8B and growing 15% annually", { kind: "observation", status: "known", confidence: "medium" }), + mkN("obs-3", "Our platform does not currently support EU data residency requirements", { kind: "observation", status: "known", confidence: "high" }), + mkN("obs-4", "Achieving compliance would require approximately 6 months and $500K engineering investment", { kind: "observation", status: "known", confidence: "medium" }), + mkN("obs-5", "Our real-time collaboration feature has no direct European equivalent", { kind: "observation", status: "provisional", confidence: "medium" }), + mkN("state-1", "Evaluating European market entry", { kind: "state", status: "provisional", confidence: "medium" }), + mkN("u-1", "Whether there is genuine demand for our category in Europe", { status: "resolved", confidence: "medium" }), + mkN("u-2", "Whether our product is suitable for European compliance requirements", { status: "resolved", confidence: "high" }), + mkN("u-3", "Whether the cost of achieving compliance is justified by the market size", { status: "resolved", confidence: "medium" }), + mkN("u-4", "Whether we have competitive differentiation against existing European players", { status: "resolved", confidence: "medium" }) + ], + resolvedNodeIds: ["u-1", "u-2", "u-3", "u-4"], activeUnknownNodeId: null, + selectedQuestion: null, noQuestionReason: "All investigation areas resolved.", + currentSummary: "European market entry is justified if compliance is achieved and the real-time collaboration feature is positioned as differentiator.", + diagnosticReasoningPattern: null + } + ], + + // SCENARIO B: Contradictory / difficult investigation — competing evidence paths + "contradictory-evidence": [ + { // Turn 0 — early, broad contradictory signals + centralStatement: "Two consultants give opposite recommendations on which supplier to choose for a $2M procurement.", + nodes: [ + mkN("obs-1", "Consultant A recommends Supplier X: lower cost, proven track record", { kind: "observation", status: "known", confidence: "high" }), + mkN("obs-2", "Consultant B recommends Supplier Y: better integration capability, higher risk but long-term upside", { kind: "observation", status: "known", confidence: "high" }), + mkN("obs-3", "Supplier X has 15+ years in the sector; Supplier Y has 2 years and mixed client reviews", { kind: "observation", status: "known", confidence: "medium" }), + mkN("obs-4", "Our current infrastructure is compatible with neither supplier out of the box", { kind: "observation", status: "known", confidence: "high" }), + mkN("state-1", "Evaluating $2M procurement against conflicting expert advice", { kind: "state", status: "provisional", confidence: "medium" }), + mkN("u-1", "Whether the conflict is genuine or reflects different evaluation criteria") + ], + resolvedNodeIds: [], activeUnknownNodeId: "u-1", + selectedQuestion: { nodeId: "u-1", question: "Are the consultants evaluating the same criteria, or are they measuring different things?", reason: "comparability_check" }, + currentSummary: "Conflicting recommendations from two experts. The first uncertainty is whether we are comparing the same dimensions.", + diagnosticReasoningPattern: "comparability_check" + }, + { // Turn 1 — one resolved, progress slow + centralStatement: "Two consultants give opposite recommendations on which supplier to choose for a $2M procurement.", + nodes: [ + mkN("obs-1", "Consultant A recommends Supplier X: lower cost, proven track record", { kind: "observation", status: "known", confidence: "high" }), + mkN("obs-2", "Consultant B recommends Supplier Y: better integration capability, higher risk but long-term upside", { kind: "observation", status: "known", confidence: "high" }), + mkN("obs-3", "Supplier X has 15+ years in the sector; Supplier Y has 2 years and mixed client reviews", { kind: "observation", status: "known", confidence: "medium" }), + mkN("obs-4", "Our current infrastructure is compatible with neither supplier out of the box", { kind: "observation", status: "known", confidence: "high" }), + mkN("obs-5", "The consultants used different evaluation weights: cost 60% vs integration 60%", { kind: "observation", status: "known", confidence: "medium" }), + mkN("state-1", "Evaluating $2M procurement against conflicting expert advice", { kind: "state", status: "provisional", confidence: "medium" }), + mkN("u-1", "Whether the conflict is genuine or reflects different evaluation criteria", { status: "resolved", confidence: "high" }), + mkN("u-2", "Which supplier's strengths align with our strategic priorities") + ], + resolvedNodeIds: ["u-1"], activeUnknownNodeId: "u-2", + selectedQuestion: { nodeId: "u-2", question: "Does cost or integration capability matter more to the organisation over a 3-year horizon?", reason: "evidence_quality" }, + currentSummary: "The conflict reflects different evaluation weights. The next uncertainty is strategic alignment.", + diagnosticReasoningPattern: "evidence_quality" + }, + { // Turn 2 — two resolved, single remaining unknown + centralStatement: "Two consultants give opposite recommendations on which supplier to choose for a $2M procurement.", + nodes: [ + mkN("obs-1", "Consultant A recommends Supplier X: lower cost, proven track record", { kind: "observation", status: "known", confidence: "high" }), + mkN("obs-2", "Consultant B recommends Supplier Y: better integration capability, higher risk but long-term upside", { kind: "observation", status: "known", confidence: "high" }), + mkN("obs-3", "Supplier X has 15+ years in the sector; Supplier Y has 2 years and mixed client reviews", { kind: "observation", status: "known", confidence: "medium" }), + mkN("obs-4", "Our current infrastructure is compatible with neither supplier out of the box", { kind: "observation", status: "known", confidence: "high" }), + mkN("obs-5", "The consultants used different evaluation weights: cost 60% vs integration 60%", { kind: "observation", status: "known", confidence: "medium" }), + mkN("obs-6", "Our strategic plan prioritises long-term capability over short-term cost savings", { kind: "observation", status: "known", confidence: "high" }), + mkN("state-1", "Evaluating $2M procurement against conflicting expert advice", { kind: "state", status: "provisional", confidence: "medium" }), + mkN("u-1", "Whether the conflict is genuine or reflects different evaluation criteria", { status: "resolved", confidence: "high" }), + mkN("u-2", "Which supplier's strengths align with our strategic priorities", { status: "resolved", confidence: "medium" }), + mkN("u-3", "Whether the integration risk of Supplier Y is manageable with internal resources") + ], + resolvedNodeIds: ["u-1", "u-2"], activeUnknownNodeId: "u-3", + selectedQuestion: { nodeId: "u-3", question: "Do we have the internal capacity to manage Supplier Y's integration risk?", reason: "alternative_explanation" }, + currentSummary: "Strategic priorities favour integration capability. The remaining uncertainty is operational feasibility.", + diagnosticReasoningPattern: "alternative_explanation" + } + ], + + // SCENARIO C: Short, early investigation — only 2 observations, unresolved + "short-early": [ + { // Turn 0 — two observations, first unknown + centralStatement: "A manufacturing company reports complaints increased by 35% while production increased by 40%.", + nodes: [ + mkN("obs-1", "Complaints increased by 35%", { kind: "observation", status: "known", confidence: "high" }), + mkN("obs-2", "Production increased by 40%", { kind: "observation", status: "known", confidence: "high" }), + mkN("state-1", "Current situation", { kind: "state", status: "provisional", confidence: "medium" }), + mkN("u-1", "Whether the two figures cover the same period") + ], + resolvedNodeIds: [], activeUnknownNodeId: "u-1", + selectedQuestion: { nodeId: "u-1", question: "Were the complaint and production figures measured over the same period?", reason: "comparability_check" }, + currentSummary: "Two changes have been reported, but we do not yet know whether the figures are directly comparable.", + diagnosticReasoningPattern: "comparability_check" + } + ] + }; +} + +/* ── Build assessment input from scenario turn ─────────── */ + +function buildAssessmentInput(turn) { + return { + situationGraph: { + centralStatement: turn.centralStatement, + currentSummary: turn.currentSummary, + nodes: turn.nodes, + edges: turn.edges || [], + activeUnknownNodeId: turn.activeUnknownNodeId, + resolvedNodeIds: turn.resolvedNodeIds + }, + selectedQuestion: turn.selectedQuestion, + noQuestionReason: turn.noQuestionReason, + diagnostics: { + promptVersion: "v0.4", + modelName: "mock-ollama", + responseDurationMs: 0, + validationStatus: "valid", + nodeCount: turn.nodes.length, + edgeCount: (turn.edges || []).length, + reasoningPattern: turn.diagnosticReasoningPattern || null + } + }; +} + +/* ── Run assessor → selector pipeline on a single turn ─── */ + +function evaluateTurn(scenarioName, scenarioTurns, turnIndex) { + const turn = scenarioTurns[turnIndex]; + const input = buildAssessmentInput(turn); + const assessment = assessInvestigationState(input); + const behaviourResult = selectBehaviour(assessment); + + return { + scenario: scenarioName, + turnNumber: turnIndex, + centralStatement: turn.centralStatement.substring(0, 60) + (turn.centralStatement.length > 60 ? "…" : ""), + assessmentPhase: assessment.phase.value, + assessmentPhaseConfidence: assessment.phase.confidence, + assessmentProgress: assessment.progress.value, + assessmentHealth: assessment.conversationHealth.value, + selectedBehaviour: behaviourResult.behaviour, + behaviourConfidence: behaviourResult.confidence, + behaviourReason: behaviourResult.reason + }; +} + +/* ── Classify selection quality ───────────────────────── */ + +function classifySelection(scenarioName, turnIndex, result) { + const { assessmentPhase, assessmentProgress, assessmentHealth, selectedBehaviour, assessmentPhaseConfidence } = result; + + if (selectedBehaviour === "acknowledge") { + // Acknowledge fires when health is healthy AND phase confidence is not low + if (assessmentHealth === "healthy" && assessmentPhaseConfidence !== "low") { + return { classification: "sensible", explanation: "Healthy conversation with established context — acknowledge fires first per priority ordering." }; + } + return { classification: "questionable", explanation: `Acknowledge fired despite health=${assessmentHealth}, phaseConf=${assessmentPhaseConfidence}.` }; + } + + if (selectedBehaviour === "clarify") { + if (assessmentHealth === "too_broad") { + return { classification: "sensible", explanation: "Too broad health justifies clarification." }; + } + if (assessmentPhase === "orienting" && result.behaviourReason?.includes("insufficient observations")) { + return { classification: "sensible", explanation: "Orienting phase with < 3 observations — clarify anchors the investigation." }; + } + return { classification: "questionable", explanation: `Clarify selected with phase=${assessmentPhase}, health=${assessmentHealth}. Verify which rule condition matched.` }; + } + + if (selectedBehaviour === "summarise") { + if (["synthesising", "concluding"].includes(assessmentPhase)) { + return { classification: "sensible", explanation: `${assessmentPhase} phase justifies a summary pass.` }; + } + return { classification: "sensible", explanation: `Summarise in ${assessmentPhase} with accumulated understanding.` }; + } + + if (selectedBehaviour === "pause") { + if ((assessmentPhase === "focusing" && assessmentProgress === "stalled") || assessmentHealth === "user_overloaded") { + return { classification: "sensible", explanation: `${assessmentPhase}+${assessmentProgress} or overloaded — pausing is correct.` }; + } + return { classification: "questionable", explanation: `Pause selected with phase=${assessmentPhase}, progress=${assessmentProgress}, health=${assessmentHealth}.` }; + } + + if (selectedBehaviour === "continue") { + // Continue is the default when no specific rule fires + if (assessmentPhase === "cannot_determine" || assessmentPhase === "exploring" || assessmentPhase === "deepening") { + return { classification: "sensible", explanation: `No specific rule matched for ${assessmentPhase} — continue to ask next question is appropriate.` }; + } + // Check if acknowledge conditions appear met but didn't fire (shouldn't happen — ack has highest priority) + if (assessmentHealth === "healthy" && assessmentPhaseConfidence !== "low") { + return { classification: "questionable", explanation: "Continue selected but acknowledge conditions appear met (healthy + confident phase). Contract mismatch?" }; + } + return { classification: "sensible", explanation: `Continue as default for ${assessmentPhase} state with no matching specific rule.` }; + } + + return { classification: "cannot determine", explanation: `Unknown behaviour: ${selectedBehaviour}` }; +} + +/* ── Test: assessor output can be passed directly into selector ─ */ + +describe("Experiment 39 — Behaviour Selection against real assessment outputs", () => { + describe("Pipeline contract", () => { + it("assessor output can be passed directly into the selector without transformation", () => { + const input = buildAssessmentInput( + getEvaluationScenarios()["short-early"][0] + ); + const assessment = assessInvestigationState(input); + const result = selectBehaviour(assessment); + expect(result.behaviour).toBeDefined(); + expect(result.confidence).toBeDefined(); + expect(result.reason).toBeDefined(); + }); + + it("every assessed turn receives one valid behaviour", () => { + const scenarios = getEvaluationScenarios(); + for (const [name, turns] of Object.entries(scenarios)) { + for (let i = 0; i < turns.length; i++) { + const input = buildAssessmentInput(turns[i]); + const assessment = assessInvestigationState(input); + const result = selectBehaviour(assessment); + expect(BEHAVIOUR_OPTIONS).toContain(result.behaviour, `${name} turn ${i}: behaviour "${result.behaviour}" not in options`); + } + } + }); + + it("repeated inputs remain deterministic", () => { + const scenarios = getEvaluationScenarios(); + for (const [name, turns] of Object.entries(scenarios)) { + for (let i = 0; i < turns.length; i++) { + const input = buildAssessmentInput(turns[i]); + const assessment = assessInvestigationState(input); + const results = Array.from({ length: 5 }, () => selectBehaviour(assessment)); + for (const r of results) { + expect(r.behaviour).toBe(results[0].behaviour); + expect(r.reason).toBe(results[0].reason); + } + } + } + }); + + it("inputs are not mutated by assessor or selector", () => { + const scenarios = getEvaluationScenarios(); + for (const [name, turns] of Object.entries(scenarios)) { + for (let i = 0; i < turns.length; i++) { + const input = buildAssessmentInput(turns[i]); + const snapshot = JSON.stringify(input); + assessInvestigationState(input); + selectBehaviour(assessInvestigationState(JSON.parse(snapshot))); + expect(JSON.stringify(input)).toBe(snapshot); + } + } + }); + }); + + describe("Three-scenario evaluation", () => { + it("evaluates all turns in long-investigation scenario", () => { + const scenarioName = "long-investigation"; + const turns = getEvaluationScenarios()[scenarioName]; + const results = []; + + for (let i = 0; i < turns.length; i++) { + const r = evaluateTurn(scenarioName, turns, i); + results.push(r); + } + + // Record full results via structured assertions + expect(results).toHaveLength(3); + + // Store for later inspection in summary tests + global._exp39_longResults = results; + + return results; + }); + + it("evaluates all turns in contradictory-evidence scenario", () => { + const scenarioName = "contradictory-evidence"; + const turns = getEvaluationScenarios()[scenarioName]; + const results = []; + + for (let i = 0; i < turns.length; i++) { + const r = evaluateTurn(scenarioName, turns, i); + results.push(r); + } + + expect(results).toHaveLength(3); + global._exp39_contradictoryResults = results; + + return results; + }); + + it("evaluates all turns in short-early scenario", () => { + const scenarioName = "short-early"; + const turns = getEvaluationScenarios()[scenarioName]; + const results = []; + + for (let i = 0; i < turns.length; i++) { + const r = evaluateTurn(scenarioName, turns, i); + results.push(r); + } + + expect(results).toHaveLength(1); + global._exp39_shortResults = results; + + return results; + }); + }); + + describe("Behaviour distribution", () => { + it("records distribution across all scenarios", () => { + const dist = { acknowledge: 0, clarify: 0, summarise: 0, pause: 0, continue: 0 }; + const allResults = [ + ...(global._exp39_longResults || []), + ...(global._exp39_contradictoryResults || []), + ...(global._exp39_shortResults || []) + ]; + + for (const r of allResults) { + dist[r.selectedBehaviour]++; + } + + expect(allResults.length).toBeGreaterThan(0); + + // Log distribution + console.log("\n=== Experiment 39 — Behaviour Distribution ==="); + console.log(`Total evaluations: ${allResults.length}`); + for (const [beh, count] of Object.entries(dist)) { + console.log(` ${beh}: ${count} (${((count/allResults.length)*100).toFixed(0)}%)`); + } + + // Store for later summary tests + global._exp39_distribution = dist; + global._exp39_allResults = allResults; + + return dist; + }); + + it("every behaviour that appears does so in a sensible context", () => { + const results = global._exp39_allResults || []; + const classifications = {}; + + for (const r of results) { + const key = `${r.scenario}-t${r.turnNumber}`; + classifications[key] = classifySelection(r.scenario, r.turnNumber, r); + } + + let questionableCount = 0; + let sensibleCount = 0; + + for (const [key, c] of Object.entries(classifications)) { + if (c.classification === "questionable") { + questionableCount++; + console.log(`\n[QUESTIONABLE] ${key}: behaviour=${r => r.selectedBehaviour}, reason: ${c.explanation}`); + } else { + sensibleCount++; + } + } + + // Log classification summary + console.log("\n=== Experiment 39 — Selection Classifications ==="); + console.log(`Sensible: ${sensibleCount}, Questionable: ${questionableCount}`); + + global._exp39_classifications = classifications; + + // Allow some questionables — we record them, don't silently correct + return classifications; + }); + }); + + describe("Scenario-level behaviour sequences", () => { + it("long-investigation produces: continue → acknowledge/summarise → summarise", () => { + const results = global._exp39_longResults || []; + + // Early state should default to Continue (no rules match) + expect(results[0].selectedBehaviour).toBeDefined(); + console.log(`\nLong turn 0: phase=${results[0].assessmentPhase}, progress=${results[0].assessmentProgress}, health=${results[0].assessmentHealth} → ${results[0].selectedBehaviour}`); + + // Deep state with resolved nodes may trigger acknowledge or summarise + expect(results[1].selectedBehaviour).toBeDefined(); + console.log(`Long turn 3: phase=${results[1].assessmentPhase}, progress=${results[1].assessmentProgress}, health=${results[1].assessmentHealth} → ${results[1].selectedBehaviour}`); + + // Terminal state should trigger summarise (concluding) + expect(results[2].selectedBehaviour).toBeDefined(); + console.log(`Long turn 4: phase=${results[2].assessmentPhase}, progress=${results[2].assessmentProgress}, health=${results[2].assessmentHealth} → ${results[2].selectedBehaviour}`); + }); + + it("contradictory-evidence produces meaningful variation", () => { + const results = global._exp39_contradictoryResults || []; + + for (let i = 0; i < results.length; i++) { + expect(results[i].selectedBehaviour).toBeDefined(); + console.log(`Contradictory turn ${i}: phase=${results[i].assessmentPhase}, progress=${results[i].assessmentProgress}, health=${results[i].assessmentHealth} → ${results[i].selectedBehaviour}`); + } + }); + + it("short-early produces appropriate single selection", () => { + const results = global._exp39_shortResults || []; + + expect(results.length).toBe(1); + console.log(`Short turn 0: phase=${results[0].assessmentPhase}, progress=${results[0].assessmentProgress}, health=${results[0].assessmentHealth} → ${results[0].selectedBehaviour}`); + }); + }); + + describe("Questionable selections preserved", () => { + it("does not silently correct questionable selections", () => { + const classifications = global._exp39_classifications || {}; + // The test passes if no assertions throw — questionable results are recorded, not fixed + expect(Object.keys(classifications).length).toBeGreaterThan(0); + }); + }); +}); + +/* ── Regression: existing assessor and selector tests pass ─ */ + +describe("Regression check", () => { + it("assessor returns sensible defaults for null input", () => { + const r = assessInvestigationState(null); + expect(r.phase.value).toBe("cannot_determine"); + expect(r.progress.value).toBe("cannot_determine"); + expect(r.conversationHealth.value).toBe("cannot_determine"); + }); + + it("selector returns continue for null input", () => { + const r = selectBehaviour(null); + expect(r.behaviour).toBe("continue"); + }); + + it("selector only outputs valid behaviours", () => { + const scenarios = getEvaluationScenarios(); + for (const [name, turns] of Object.entries(scenarios)) { + for (const turn of turns) { + const input = buildAssessmentInput(turn); + const assessment = assessInvestigationState(input); + const result = selectBehaviour(assessment); + expect(BEHAVIOUR_OPTIONS).toContain(result.behaviour); + } + } + }); +});