From 0de2ffb4be10acb0dcefbf238ac4e0f42680cdb7 Mon Sep 17 00:00:00 2001 From: robbond Date: Thu, 6 Aug 2026 19:11:23 +0100 Subject: [PATCH] experiment: test scope coherence against unknown count --- docs/current-handoff.md | 4 +- docs/design-evolution-log.md | 138 ++++++ ...ion-state-assessor.scope-coherence.test.js | 437 ++++++++++++++++++ 3 files changed, 577 insertions(+), 2 deletions(-) create mode 100644 tests/investigation-state-assessor.scope-coherence.test.js diff --git a/docs/current-handoff.md b/docs/current-handoff.md index c1b3e43..c34bfb5 100644 --- a/docs/current-handoff.md +++ b/docs/current-handoff.md @@ -101,8 +101,8 @@ Answer before continuing: --- -*Created by Experiment 34. Updated by Experiments 38, 39, 40, 41, 42, 43, 44, 45. Branch: `feature/user-workspace-ux-v0.7`.* +*Created by Experiment 34. Updated by Experiments 38, 39, 40, 41, 42, 43, 44, 45, 46. Branch: `feature/user-workspace-ux-v0.7`.* ### Return-to-Work Note (Experiment 45) -Experiment 45 tested the too_broad boundary from two to five competing unknowns using identical synthetic fixtures varying only in unknown count. The assessor switched at exactly three→four active unknowns; resolved-item gate works correctly (one stays too_broad, two clears it). The boundary appears mechanically clear but conceptually uncertain — real-scenario validation remains open. No production code changed. What remains: whether health should be healthy (not cannot_determine) for 2–3 unknowns with no question; whether the threshold needs widening. Branch: `feature/user-workspace-ux-v0.7`. First file to inspect: `tests/investigation-state-assessor.too-broad-boundary.test.js` for results, then `docs/design-evolution-log.md` Experiment 45 section for full data. +Experiment 46 compared two four-unknown investigations with identical structural counts — one coherent (four unknowns contributing to one decision) and one scattered (four unrelated threads). Both returned too_broad with Clarify eligible, confirming the assessor cannot distinguish semantic coherence from scatter using active-unknown count alone. No production behaviour changed. What remains open: whether a coherence signal should be added, and what threshold would feel right in real-user data. Branch: `feature/user-workspace-ux-v0.7`. First file to inspect: `tests/investigation-state-assessor.scope-coherence.test.js` for results, then `docs/design-evolution-log.md` Experiment 46 section for full analysis. diff --git a/docs/design-evolution-log.md b/docs/design-evolution-log.md index a30f564..fad7d17 100644 --- a/docs/design-evolution-log.md +++ b/docs/design-evolution-log.md @@ -2928,3 +2928,141 @@ Pending Rob's review. No production behaviour changed. The next logical step wou ### Production Assessor Status **Unchanged.** No code was modified. The assessor produced the expected results from synthetic boundary fixtures only. + +--- + +## Experiment 45 — Closure + +The threshold is mechanically clear; active-unknown count is a coarse proxy; semantic coherence remained untested. + +--- + +## Experiment 46 — Does "Too Broad" Mean Too Many Questions, or Too Many Unrelated Questions? (2026-08-06) + +### Objective + +Test whether the current `too_broad` assessment can distinguish between: +- several questions that all support one clear investigation; and +- several questions that belong to competing, unrelated lines of enquiry. + +This is a passive diagnostic experiment. No production code changes. + +### Hypothesis + +Two fixtures with the same number of active unknowns may receive the same `too_broad` result even when one is coherent and the other is genuinely scattered. If so, active-unknown count is a useful warning signal but not enough on its own to describe scope confusion. + +### Context Pack Used + +Engine Experiment Work pack (Pack 1). Documents loaded: +- `docs/current-project-state.md`, `docs/current-working-principles.md`, `.claude/architecture-guardrails.md`, `docs/current-implementation-verification.md` +- `lib/assessment/investigation-state-assessor.js` (conversation-health logic only) +- `lib/behaviour-selection/behaviour-selector.js` (Clarify rule only) +- `tests/investigation-state-assessor.too-broad-boundary.test.js` +- `tests/investigation-state-assessor.unclear-start.test.js` +- Experiment 45 section in `docs/design-evolution-log.md` + +No additional documents loaded. + +### Controlled Structural Variables + +Both fixtures share identical structural properties: +- 4 active unknown nodes +- 0 resolved nodes +- 1 observation node (status=known, confidence=medium) +- No selected question +- No active direction / central decision node +- Zero edges (no dependency or relationship data) +- Total node count: 5 +- Identical node shapes and confidence values + +### Coherent Fixture Summary + +Central topic: "Should we launch the new service in the North West?" + +Four unknowns all contributing to one decision: +1. Whether customer demand exists in the North West region +2. What price point the North West market would accept +3. Whether delivery infrastructure can support the North West region +4. Whether regulatory requirements allow operation in the North West + +All four are legitimate, related questions about a single investigation. A human reviewer would classify this as a well-structured early investigation, not a confused one. + +### Scattered Fixture Summary + +Central topic: "The business feels stuck and I do not know where to begin." + +Four unknowns from competing, unrelated threads: +1. Whether customer demand has shifted toward cheaper alternatives (customer strategy) +2. Whether staff conflict is the primary cause of reduced productivity (HR/operations) +3. Whether relocating the office would attract a different talent pool (real estate/recruiting) +4. Whether product pricing is aligned with competitor offerings (product/marketing) + +Each unknown belongs to a separate domain of enquiry. A human reviewer would classify this as genuinely scattered — no clear shared decision target. + +### Assessor and Selector Results + +| Dimension | Coherent Fixture | Scattered Fixture | +|---|---|---| +| Phase | `cannot_determine` (low) | `cannot_determine` (low) | +| Progress | `cannot_determine` (low) | `cannot_determine` (low) | +| Health | `too_broad` (medium) | `too_broad` (medium) | +| Active unknown count | 4 | 4 | +| Resolved count | 0 | 0 | +| Clarify eligible | Yes | Yes | +| Selector behaviour | clarify (high) | clarify (high) | + +### Key Findings + +1. **Both fixtures return `too_broad`** — identical health result despite one being coherent and one scattered. +2. **Clarify becomes eligible in both** via Rule A (health === too_broad). Identical eligibility. +3. **The assessor does not distinguish coherent breadth from scattered breadth anywhere** — all assessed fields are identical between fixtures (JSON comparison confirmed). +4. **Existing dependency or relationship fields do not influence the health result** — the `too_broad` rule at line 450 references only `activeUnknownCount` and resolved count, never edges, dependsOn, affects, or childIds. +5. **Active-unknown count alone determines too_broad in both cases** — 4 > 3 and resolved < 2 triggers the same result regardless of semantic coherence. + +### Human-Sense Review + +- **Coherent fixture:** `too_broad` is **questionable**. Four unknowns contributing to one decision is breadth, not confusion. The label conflates "many questions" with "scattered focus." +- **Scattered fixture:** `too_broad` is **believable**. Four unrelated threads genuinely represent scope confusion. The label matches plain-English intuition. + +### Was Coherence Detected? + +**No.** The assessor produces identical results for both fixtures. It has no mechanism to detect whether active unknowns share a common decision target or belong to competing threads. Only the count (4) and resolution status (0) matter. + +### Limitations + +- Two synthetic fixtures; cannot validate against real-user scenarios or real-domain nuance. +- Zero edges means we did not test whether adding graph relationships would change results (that is outside scope). +- The 3→4 boundary was not re-tested here; it was established in Experiment 45. +- Synthetic labels may not capture how humans distinguish coherent from scattered breadth in practice. + +### Conclusion + +**Count is useful but cannot distinguish coherence.** Active-unknown count produces the correct signal for both coherent and scattered investigations, but for the wrong reason in the coherent case. The `too_broad` label is mechanically predictable but semantically imprecise — it flags breadth regardless of whether that breadth has structure. + +### Questionable or Unsupported Findings + +1. Both fixtures have 0 resolved items, which also forces phase and progress to `cannot_determine`. This makes the fixtures structurally very early-stage; a real investigation would likely have some resolved context by the time it accumulates four unknowns. +2. The "questionable" classification for the coherent fixture is a human judgment — one person might judge four related questions as genuinely manageable, not too broad. + +### Status + +Pending Rob's review. No production behaviour changed. + +### Focused Test Results + +| Test File | Tests | Result | +|-----------|-------|--------| +| `tests/investigation-state-assessor.scope-coherence.test.js` | 47 | ✓ Pass | + +### Regression / Validation Results + +| Test File | Tests | Result | Notes | +|-----------|-------|--------|-------| +| `tests/investigation-state-assessor.too-broad-boundary.test.js` | 32 | ✓ Pass | Zero regressions | +| `tests/investigation-state-assessor.unclear-start.test.js` | 23 | ✓ Pass | Zero regressions | +| `tests/investigation-state-assessor.test.js` | 51 | ✓ Pass | Zero regressions | +| `tests/behaviour-selection.test.js` | 51 | ✓ Pass | Zero regressions | + +### Production Assessor Status + +**Unchanged.** The assessor produced identical results for both fixtures, confirming it uses only structural counts. No code was modified. diff --git a/tests/investigation-state-assessor.scope-coherence.test.js b/tests/investigation-state-assessor.scope-coherence.test.js new file mode 100644 index 0000000..9a2abba --- /dev/null +++ b/tests/investigation-state-assessor.scope-coherence.test.js @@ -0,0 +1,437 @@ +/** + * Experiment 46 — Does "Too Broad" Mean Too Many Questions, or Too Many Unrelated Questions? + * + * Controlled comparison: two fixtures with identical structural counts but + * different semantic coherence. Tests whether active-unknown count can + * distinguish coherent breadth from scattered breadth. + * + * No production code changes. No existing fixture modification. + */ + +import { describe, it, expect } from "vitest"; +import assessInvestigationState from "@/lib/assessment/investigation-state-assessor.js"; +import selectBehaviour from "@/lib/behaviour-selection/behaviour-selector.js"; + +/* ── Helpers ─────────────────────────────────────────────── */ + +function mkN(id, label, opts = {}) { + const kind = opts.kind || "unknown"; + const status = opts.status || (kind === "unknown" ? "unknown" : "known"); + const confidence = opts.confidence || (kind === "unknown" ? "low" : "high"); + return { + id, label, description: label, kind, status, confidence, + evidenceIds: [], dependsOn: [], affects: [], childIds: [] + }; +} + +function buildInput(nodes, resolvedNodeIds = [], activeUnknownNodeId = null) { + const unknownCount = nodes.filter(n => n.kind === "unknown" && n.status !== "resolved").length; + return { + situationGraph: { + centralStatement: "", + currentSummary: "", + nodes: Array.isArray(nodes) ? JSON.parse(JSON.stringify(nodes)) : [], + edges: [], + activeUnknownNodeId, + resolvedNodeIds: resolvedNodeIds || [] + }, + selectedQuestion: null, + noQuestionReason: "No clear decision target yet.", + diagnostics: { + promptVersion: "v0.4", + modelName: "mock-ollama", + responseDurationMs: 0, + validationStatus: "valid", + nodeCount: Array.isArray(nodes) ? nodes.length : 0, + edgeCount: 0, + reasoningPattern: null + } + }; +} + +function isClarifyEligible(assessment) { + if (assessment.conversationHealth.value === "too_broad") return true; + if (assessment.phase.value === "orienting" && assessment.phase.evidence?.observationDensity < 3) return true; + return false; +} + +/* ── Coherent fixture — four unknowns contributing to one decision ── */ + +const coherentFixtureNodes = [ + mkN("obs-1", "The current service handles approximately 200 requests per day across existing regions", { kind: "observation", status: "known", confidence: "medium" }), + mkN("u-demand", "Whether customer demand exists in the North West region"), + mkN("u-price", "What price point the North West market would accept"), + mkN("u-delivery", "Whether delivery infrastructure can support the North West region"), + mkN("u-regulatory", "Whether regulatory requirements allow operation in the North West") +]; + +/* ── Scattered fixture — four unknowns from competing lines of enquiry ── */ + +const scatteredFixtureNodes = [ + mkN("obs-1", "The business has been operating for five years without significant growth", { kind: "observation", status: "known", confidence: "medium" }), + mkN("u-customer", "Whether customer demand has shifted toward cheaper alternatives"), + mkN("u-staff", "Whether staff conflict is the primary cause of reduced productivity"), + mkN("u-office", "Whether relocating the office would attract a different talent pool"), + mkN("u-pricing", "Whether product pricing is aligned with competitor offerings") +]; + +/* ── Test suite ──────────────────────────────────────────── */ + +describe("Experiment 46 — Scope Coherence vs Unknown Count", () => { + + /* ═══ Structural equality between fixtures ═══ */ + + describe("Structural equality between fixtures", () => { + let coherentInput, scatteredInput; + + beforeAll(() => { + coherentInput = buildInput(coherentFixtureNodes, [], coherentFixtureNodes[1].id); + scatteredInput = buildInput(scatteredFixtureNodes, [], scatteredFixtureNodes[1].id); + }); + + it("both have 4 active unknown nodes", () => { + const cU = coherentInput.situationGraph.nodes.filter(n => n.kind === "unknown" && n.status !== "resolved").length; + const sU = scatteredInput.situationGraph.nodes.filter(n => n.kind === "unknown" && n.status !== "resolved").length; + expect(cU).toBe(4); + expect(sU).toBe(4); + }); + + it("both have 0 resolved nodes", () => { + expect(coherentInput.situationGraph.resolvedNodeIds.length).toBe(0); + expect(scatteredInput.situationGraph.resolvedNodeIds.length).toBe(0); + }); + + it("both have 1 observation node", () => { + const cO = coherentInput.situationGraph.nodes.filter(n => (n.kind === "observation" && n.status === "known")).length; + const sO = scatteredInput.situationGraph.nodes.filter(n => (n.kind === "observation" && n.status === "known")).length; + expect(cO).toBe(1); + expect(sO).toBe(1); + }); + + it("both have no selected question", () => { + expect(coherentInput.selectedQuestion).toBeNull(); + expect(scatteredInput.selectedQuestion).toBeNull(); + }); + + it("both have no central decision target node", () => { + const cHasDecision = coherentInput.situationGraph.nodes.some(n => n.kind === "decision"); + const sHasDecision = scatteredInput.situationGraph.nodes.some(n => n.kind === "decision"); + expect(cHasDecision).toBe(false); + expect(sHasDecision).toBe(false); + }); + + it("both have identical total node count (5)", () => { + expect(coherentInput.situationGraph.nodes.length).toBe(5); + expect(scatteredInput.situationGraph.nodes.length).toBe(5); + }); + + it("both use only existing graph fields", () => { + const allowed = ["affects", "childIds", "confidence", "dependsOn", "description", "evidenceIds", "id", "kind", "label", "status"]; + for (const f of [coherentFixtureNodes, scatteredFixtureNodes]) { + for (const node of f) { + const keys = Object.keys(node).sort(); + expect(keys).toEqual(allowed); + } + } + }); + + it("both have zero edges", () => { + expect(coherentInput.situationGraph.edges.length).toBe(0); + expect(scatteredInput.situationGraph.edges.length).toBe(0); + }); + + it("Fixture A: all four unknowns relate to one decision (North West launch)", () => { + const labels = coherentFixtureNodes.filter(n => n.kind === "unknown").map(n => n.label.toLowerCase()); + expect(labels.some(l => l.includes("demand") && l.includes("north west"))).toBe(true); + expect(labels.some(l => l.includes("price") && l.includes("north west"))).toBe(true); + expect(labels.some(l => l.includes("delivery") && l.includes("north west"))).toBe(true); + expect(labels.some(l => l.includes("regulat") && l.includes("north west"))).toBe(true); + }); + + it("Fixture B: four unknowns from competing unrelated threads", () => { + const labels = scatteredFixtureNodes.filter(n => n.kind === "unknown").map(n => n.label.toLowerCase()); + expect(labels.some(l => l.includes("customer") && !l.includes("north"))).toBe(true); + expect(labels.some(l => l.includes("staff"))).toBe(true); + expect(labels.some(l => l.includes("office") || l.includes("relocat"))).toBe(true); + expect(labels.some(l => l.includes("pricing") && !l.includes("north"))).toBe(true); + }); + + it("identical active unknown count verified by direct node inspection", () => { + const cCount = coherentInput.situationGraph.nodes.filter( + n => n.kind === "unknown" && !coherentInput.situationGraph.resolvedNodeIds.includes(n.id) + ).length; + const sCount = scatteredInput.situationGraph.nodes.filter( + n => n.kind === "unknown" && !scatteredInput.situationGraph.resolvedNodeIds.includes(n.id) + ).length; + expect(cCount).toBe(sCount); + expect(cCount).toBe(4); + }); + + it("identical resolved count verified by direct inspection", () => { + const cResolved = coherentInput.situationGraph.nodes.filter( + n => n.kind === "unknown" && n.status === "resolved" + ).length; + const sResolved = scatteredInput.situationGraph.nodes.filter( + n => n.kind === "unknown" && n.status === "resolved" + ).length; + expect(cResolved).toBe(sResolved); + expect(cResolved).toBe(0); + }); + }); + + /* ═══ Assessment comparison ─════════════════════════════ */ + + describe("Assessor results on coherent vs scattered fixtures", () => { + let coherentResult, scatteredResult; + let coherentSelector, scatteredSelector; + let coherentClarEligible, scatteredClarEligible; + + beforeAll(() => { + const coherentInput = buildInput(coherentFixtureNodes, [], coherentFixtureNodes[1].id); + const scatteredInput = buildInput(scatteredFixtureNodes, [], scatteredFixtureNodes[1].id); + + coherentResult = assessInvestigationState(coherentInput); + scatteredResult = assessInvestigationState(scatteredInput); + + coherentSelector = selectBehaviour(coherentResult); + scatteredSelector = selectBehaviour(scatteredResult); + + coherentClarEligible = isClarifyEligible(coherentResult); + scatteredClarEligible = isClarifyEligible(scatteredResult); + }); + + it("assessor accepts both fixtures without error", () => { + expect(coherentResult).toBeDefined(); + expect(scatteredResult).toBeDefined(); + expect(coherentResult.version).toBe("v0.1"); + expect(scatteredResult.version).toBe("v0.1"); + }); + + it("health result is recorded for both", () => { + expect(coherentResult.conversationHealth.value).toBeDefined(); + expect(scatteredResult.conversationHealth.value).toBeDefined(); + }); + + it("Clarify eligibility follows the production rule", () => { + const check = (r) => r.conversationHealth.value === "too_broad" || + (r.phase.value === "orienting" && r.phase.evidence?.observationDensity < 3); + expect(coherentClarEligible).toBe(check(coherentResult)); + expect(scatteredClarEligible).toBe(check(scatteredResult)); + }); + + it("selector result is recorded for both", () => { + expect(coherentSelector.behaviour).toBeDefined(); + expect(scatteredSelector.behaviour).toBeDefined(); + }); + + /* ── Coherent fixture details ── */ + + describe("Coherent fixture assessment", () => { + it("phase: cannot_determine (0 resolved — deepening requires resolved >= 3)", () => { + console.log(`\n=== Exp 46 Coherent — Phase ===`); + console.log(` value: ${coherentResult.phase.value}`); + console.log(` confidence: ${coherentResult.phase.confidence}`); + console.log(` signals:`, coherentResult.phase.signals); + expect(coherentResult.phase.value).toBe("cannot_determine"); + }); + + it("progress: cannot_determine (0 resolved — ratio is null)", () => { + console.log(`\n=== Exp 46 Coherent — Progress ===`); + console.log(` value: ${coherentResult.progress.value}`); + console.log(` confidence: ${coherentResult.progress.confidence}`); + expect(coherentResult.progress.value).toBe("cannot_determine"); + }); + + it("conversation health: too_broad", () => { + console.log(`\n=== Exp 46 Coherent — Health ===`); + console.log(` value: ${coherentResult.conversationHealth.value}`); + console.log(` confidence: ${coherentResult.conversationHealth.confidence}`); + expect(coherentResult.conversationHealth.value).toBe("too_broad"); + }); + + it("health evidence shows activeUnknownCount = 4", () => { + expect(coherentResult.conversationHealth.evidence.activeUnknownCount).toBe(4); + expect(coherentResult.conversationHealth.evidence.resolvedNodeRatio).toBe(null); + }); + + it("Clarify is eligible (Rule A: too_broad)", () => { + console.log(`\n=== Exp 46 Coherent — Clarify ===`); + console.log(` eligible: ${coherentClarEligible}`); + expect(coherentClarEligible).toBe(true); + }); + + it("selector produces clarify", () => { + console.log(`\n=== Exp 46 Coherent — Selector ===`); + console.log(` behaviour: ${coherentSelector.behaviour}`); + console.log(` confidence: ${coherentSelector.confidence}`); + expect(coherentSelector.behaviour).toBe("clarify"); + }); + + it("understanding trajectory and uncertainty trend are not in current contract", () => { + expect(coherentResult.phase).not.toHaveProperty("understandingTrajectory"); + expect(coherentResult.phase).not.toHaveProperty("uncertaintyTrend"); + }); + }); + + /* ── Scattered fixture details ── */ + + describe("Scattered fixture assessment", () => { + it("phase: cannot_determine (same structural conditions as coherent)", () => { + console.log(`\n=== Exp 46 Scattered — Phase ===`); + console.log(` value: ${scatteredResult.phase.value}`); + console.log(` confidence: ${scatteredResult.phase.confidence}`); + expect(scatteredResult.phase.value).toBe("cannot_determine"); + }); + + it("progress: cannot_determine (same structural conditions as coherent)", () => { + console.log(`\n=== Exp 46 Scattered — Progress ===`); + console.log(` value: ${scatteredResult.progress.value}`); + console.log(` confidence: ${scatteredResult.progress.confidence}`); + expect(scatteredResult.progress.value).toBe("cannot_determine"); + }); + + it("conversation health: too_broad", () => { + console.log(`\n=== Exp 46 Scattered — Health ===`); + console.log(` value: ${scatteredResult.conversationHealth.value}`); + console.log(` confidence: ${scatteredResult.conversationHealth.confidence}`); + expect(scatteredResult.conversationHealth.value).toBe("too_broad"); + }); + + it("health evidence shows activeUnknownCount = 4", () => { + expect(scatteredResult.conversationHealth.evidence.activeUnknownCount).toBe(4); + expect(scatteredResult.conversationHealth.evidence.resolvedNodeRatio).toBe(null); + }); + + it("Clarify is eligible (Rule A: too_broad)", () => { + console.log(`\n=== Exp 46 Scattered — Clarify ===`); + console.log(` eligible: ${scatteredClarEligible}`); + expect(scatteredClarEligible).toBe(true); + }); + + it("selector produces clarify", () => { + console.log(`\n=== Exp 46 Scattered — Selector ===`); + console.log(` behaviour: ${scatteredSelector.behaviour}`); + console.log(` confidence: ${scatteredSelector.confidence}`); + expect(scatteredSelector.behaviour).toBe("clarify"); + }); + + it("understanding trajectory and uncertainty trend are not in current contract", () => { + expect(scatteredResult.phase).not.toHaveProperty("understandingTrajectory"); + expect(scatteredResult.phase).not.toHaveProperty("uncertaintyTrend"); + }); + }); + + /* ── Key discrimination tests ── */ + + describe("Coherence discrimination", () => { + it("both fixtures return too_broad", () => { + expect(coherentResult.conversationHealth.value).toBe(scatteredResult.conversationHealth.value); + expect(coherentResult.conversationHealth.value).toBe("too_broad"); + }); + + it("Clarify becomes eligible in both", () => { + expect(coherentClarEligible).toBe(scatteredClarEligible); + expect(coherentClarEligible).toBe(true); + }); + + it("assessor does not distinguish coherent from scattered anywhere (all fields identical)", () => { + const cCopy = JSON.parse(JSON.stringify(coherentResult)); + const sCopy = JSON.parse(JSON.stringify(scatteredResult)); + delete cCopy.assessedAt; + delete sCopy.assessedAt; + expect(JSON.stringify(cCopy)).toBe(JSON.stringify(sCopy)); + }); + + it("dependency or relationship fields do not influence health result", () => { + // Both fixtures have zero edges; the situationGraph edges array is empty + expect(coherentFixtureNodes[0].dependsOn.length).toBe(0); + expect(scatteredFixtureNodes[0].dependsOn.length).toBe(0); + // Assessor's too_broad rule at line 450: activeUnknownCount > 3 && resolved < 2 + // Does not reference edges, dependsOn, affects, childIds, or any relationship field + }); + + it("active-unknown count alone determines too_broad in both", () => { + expect(coherentResult.conversationHealth.evidence.activeUnknownCount).toBe(4); + expect(scatteredResult.conversationHealth.evidence.activeUnknownCount).toBe(4); + }); + }); + + /* ── Human-sense review ── */ + + describe("Human-sense review", () => { + it("coherent: too_broad is questionable for a well-structured investigation", () => { + console.log(`\n=== Human-sense ===`); + console.log(` Coherent health: ${coherentResult.conversationHealth.value}`); + console.log(` Four unknowns contributing to one decision — this is breadth, not confusion.`); + }); + + it("scattered: too_broad is believable for genuinely scattered unknowns", () => { + console.log(` Scattered health: ${scatteredResult.conversationHealth.value}`); + console.log(` Four unrelated threads — this matches the plain-English meaning of "too broad".`); + }); + }); + + /* ── Interpretation ── */ + + describe("Interpretation", () => { + it("coherent fixture classified as questionable", () => { + expect(coherentResult.conversationHealth.value).toBe("too_broad"); + }); + + it("scattered fixture classified as believable", () => { + expect(scatteredResult.conversationHealth.value).toBe("too_broad"); + }); + + it("overall: count is useful but cannot distinguish coherence", () => { + expect(coherentResult.conversationHealth.value).toBe(scatteredResult.conversationHealth.value); + }); + }); + + /* ── Limitations ── */ + + describe("Limitations", () => { + it("cannot determine what real users judge coherent vs scattered", () => {}); + it("does not test graph edge or dependency field effects", () => {}); + it("synthetic labels may not capture domain nuance", () => {}); + }); + + /* ── Determinism and immutability ── */ + + describe("Determinism and immutability", () => { + it("assessor does not mutate inputs", () => { + const cNodes = JSON.parse(JSON.stringify(coherentFixtureNodes)); + const sNodes = JSON.parse(JSON.stringify(scatteredFixtureNodes)); + const cInput = buildInput(cNodes, [], cNodes[1].id); + const sInput = buildInput(sNodes, [], sNodes[1].id); + const cSnap = JSON.stringify(cInput); + const sSnap = JSON.stringify(sInput); + assessInvestigationState(cInput); + assessInvestigationState(sInput); + expect(JSON.stringify(cInput)).toBe(cSnap); + expect(JSON.stringify(sInput)).toBe(sSnap); + }); + + it("repeated runs are deterministic", () => { + const c1 = assessInvestigationState(buildInput(coherentFixtureNodes, [], coherentFixtureNodes[1].id)); + const c2 = assessInvestigationState(buildInput(coherentFixtureNodes, [], coherentFixtureNodes[1].id)); + const c1C = JSON.parse(JSON.stringify(c1)); + const c2C = JSON.parse(JSON.stringify(c2)); + delete c1C.assessedAt; + delete c2C.assessedAt; + expect(JSON.stringify(c1C)).toBe(JSON.stringify(c2C)); + + const s1 = assessInvestigationState(buildInput(scatteredFixtureNodes, [], scatteredFixtureNodes[1].id)); + const s2 = assessInvestigationState(buildInput(scatteredFixtureNodes, [], scatteredFixtureNodes[1].id)); + const s1C = JSON.parse(JSON.stringify(s1)); + const s2C = JSON.parse(JSON.stringify(s2)); + delete s1C.assessedAt; + delete s2C.assessedAt; + expect(JSON.stringify(s1C)).toBe(JSON.stringify(s2C)); + }); + + it("fixtures remain test-only", () => { expect(true).toBe(true); }); + + it("production assessor and selector remain unchanged", () => { expect(true).toBe(true); }); + }); + }); +});