/** * Experiment 46 — Does "Too Broad" Mean Too Many Questions, or Too Many Unrelated Questions? * * Controlled comparison: two fixtures with identical structural counts but * different semantic coherence. Tests whether active-unknown count can * distinguish coherent breadth from scattered breadth. * * No production code changes. No existing fixture modification. */ import { describe, it, expect } from "vitest"; import assessInvestigationState from "@/lib/assessment/investigation-state-assessor.js"; import selectBehaviour from "@/lib/behaviour-selection/behaviour-selector.js"; /* ── Helpers ─────────────────────────────────────────────── */ function mkN(id, label, opts = {}) { const kind = opts.kind || "unknown"; const status = opts.status || (kind === "unknown" ? "unknown" : "known"); const confidence = opts.confidence || (kind === "unknown" ? "low" : "high"); return { id, label, description: label, kind, status, confidence, evidenceIds: [], dependsOn: [], affects: [], childIds: [] }; } function buildInput(nodes, resolvedNodeIds = [], activeUnknownNodeId = null) { const unknownCount = nodes.filter(n => n.kind === "unknown" && n.status !== "resolved").length; return { situationGraph: { centralStatement: "", currentSummary: "", nodes: Array.isArray(nodes) ? JSON.parse(JSON.stringify(nodes)) : [], edges: [], activeUnknownNodeId, resolvedNodeIds: resolvedNodeIds || [] }, selectedQuestion: null, noQuestionReason: "No clear decision target yet.", diagnostics: { promptVersion: "v0.4", modelName: "mock-ollama", responseDurationMs: 0, validationStatus: "valid", nodeCount: Array.isArray(nodes) ? nodes.length : 0, edgeCount: 0, reasoningPattern: null } }; } function isClarifyEligible(assessment) { if (assessment.conversationHealth.value === "too_broad") return true; if (assessment.phase.value === "orienting" && assessment.phase.evidence?.observationDensity < 3) return true; return false; } /* ── Coherent fixture — four unknowns contributing to one decision ── */ const coherentFixtureNodes = [ mkN("obs-1", "The current service handles approximately 200 requests per day across existing regions", { kind: "observation", status: "known", confidence: "medium" }), mkN("u-demand", "Whether customer demand exists in the North West region"), mkN("u-price", "What price point the North West market would accept"), mkN("u-delivery", "Whether delivery infrastructure can support the North West region"), mkN("u-regulatory", "Whether regulatory requirements allow operation in the North West") ]; /* ── Scattered fixture — four unknowns from competing lines of enquiry ── */ const scatteredFixtureNodes = [ mkN("obs-1", "The business has been operating for five years without significant growth", { kind: "observation", status: "known", confidence: "medium" }), mkN("u-customer", "Whether customer demand has shifted toward cheaper alternatives"), mkN("u-staff", "Whether staff conflict is the primary cause of reduced productivity"), mkN("u-office", "Whether relocating the office would attract a different talent pool"), mkN("u-pricing", "Whether product pricing is aligned with competitor offerings") ]; /* ── Test suite ──────────────────────────────────────────── */ describe("Experiment 46 — Scope Coherence vs Unknown Count", () => { /* ═══ Structural equality between fixtures ═══ */ describe("Structural equality between fixtures", () => { let coherentInput, scatteredInput; beforeAll(() => { coherentInput = buildInput(coherentFixtureNodes, [], coherentFixtureNodes[1].id); scatteredInput = buildInput(scatteredFixtureNodes, [], scatteredFixtureNodes[1].id); }); it("both have 4 active unknown nodes", () => { const cU = coherentInput.situationGraph.nodes.filter(n => n.kind === "unknown" && n.status !== "resolved").length; const sU = scatteredInput.situationGraph.nodes.filter(n => n.kind === "unknown" && n.status !== "resolved").length; expect(cU).toBe(4); expect(sU).toBe(4); }); it("both have 0 resolved nodes", () => { expect(coherentInput.situationGraph.resolvedNodeIds.length).toBe(0); expect(scatteredInput.situationGraph.resolvedNodeIds.length).toBe(0); }); it("both have 1 observation node", () => { const cO = coherentInput.situationGraph.nodes.filter(n => (n.kind === "observation" && n.status === "known")).length; const sO = scatteredInput.situationGraph.nodes.filter(n => (n.kind === "observation" && n.status === "known")).length; expect(cO).toBe(1); expect(sO).toBe(1); }); it("both have no selected question", () => { expect(coherentInput.selectedQuestion).toBeNull(); expect(scatteredInput.selectedQuestion).toBeNull(); }); it("both have no central decision target node", () => { const cHasDecision = coherentInput.situationGraph.nodes.some(n => n.kind === "decision"); const sHasDecision = scatteredInput.situationGraph.nodes.some(n => n.kind === "decision"); expect(cHasDecision).toBe(false); expect(sHasDecision).toBe(false); }); it("both have identical total node count (5)", () => { expect(coherentInput.situationGraph.nodes.length).toBe(5); expect(scatteredInput.situationGraph.nodes.length).toBe(5); }); it("both use only existing graph fields", () => { const allowed = ["affects", "childIds", "confidence", "dependsOn", "description", "evidenceIds", "id", "kind", "label", "status"]; for (const f of [coherentFixtureNodes, scatteredFixtureNodes]) { for (const node of f) { const keys = Object.keys(node).sort(); expect(keys).toEqual(allowed); } } }); it("both have zero edges", () => { expect(coherentInput.situationGraph.edges.length).toBe(0); expect(scatteredInput.situationGraph.edges.length).toBe(0); }); it("Fixture A: all four unknowns relate to one decision (North West launch)", () => { const labels = coherentFixtureNodes.filter(n => n.kind === "unknown").map(n => n.label.toLowerCase()); expect(labels.some(l => l.includes("demand") && l.includes("north west"))).toBe(true); expect(labels.some(l => l.includes("price") && l.includes("north west"))).toBe(true); expect(labels.some(l => l.includes("delivery") && l.includes("north west"))).toBe(true); expect(labels.some(l => l.includes("regulat") && l.includes("north west"))).toBe(true); }); it("Fixture B: four unknowns from competing unrelated threads", () => { const labels = scatteredFixtureNodes.filter(n => n.kind === "unknown").map(n => n.label.toLowerCase()); expect(labels.some(l => l.includes("customer") && !l.includes("north"))).toBe(true); expect(labels.some(l => l.includes("staff"))).toBe(true); expect(labels.some(l => l.includes("office") || l.includes("relocat"))).toBe(true); expect(labels.some(l => l.includes("pricing") && !l.includes("north"))).toBe(true); }); it("identical active unknown count verified by direct node inspection", () => { const cCount = coherentInput.situationGraph.nodes.filter( n => n.kind === "unknown" && !coherentInput.situationGraph.resolvedNodeIds.includes(n.id) ).length; const sCount = scatteredInput.situationGraph.nodes.filter( n => n.kind === "unknown" && !scatteredInput.situationGraph.resolvedNodeIds.includes(n.id) ).length; expect(cCount).toBe(sCount); expect(cCount).toBe(4); }); it("identical resolved count verified by direct inspection", () => { const cResolved = coherentInput.situationGraph.nodes.filter( n => n.kind === "unknown" && n.status === "resolved" ).length; const sResolved = scatteredInput.situationGraph.nodes.filter( n => n.kind === "unknown" && n.status === "resolved" ).length; expect(cResolved).toBe(sResolved); expect(cResolved).toBe(0); }); }); /* ═══ Assessment comparison ─════════════════════════════ */ describe("Assessor results on coherent vs scattered fixtures", () => { let coherentResult, scatteredResult; let coherentSelector, scatteredSelector; let coherentClarEligible, scatteredClarEligible; beforeAll(() => { const coherentInput = buildInput(coherentFixtureNodes, [], coherentFixtureNodes[1].id); const scatteredInput = buildInput(scatteredFixtureNodes, [], scatteredFixtureNodes[1].id); coherentResult = assessInvestigationState(coherentInput); scatteredResult = assessInvestigationState(scatteredInput); coherentSelector = selectBehaviour(coherentResult); scatteredSelector = selectBehaviour(scatteredResult); coherentClarEligible = isClarifyEligible(coherentResult); scatteredClarEligible = isClarifyEligible(scatteredResult); }); it("assessor accepts both fixtures without error", () => { expect(coherentResult).toBeDefined(); expect(scatteredResult).toBeDefined(); expect(coherentResult.version).toBe("v0.1"); expect(scatteredResult.version).toBe("v0.1"); }); it("health result is recorded for both", () => { expect(coherentResult.conversationHealth.value).toBeDefined(); expect(scatteredResult.conversationHealth.value).toBeDefined(); }); it("Clarify eligibility follows the production rule", () => { const check = (r) => r.conversationHealth.value === "too_broad" || (r.phase.value === "orienting" && r.phase.evidence?.observationDensity < 3); expect(coherentClarEligible).toBe(check(coherentResult)); expect(scatteredClarEligible).toBe(check(scatteredResult)); }); it("selector result is recorded for both", () => { expect(coherentSelector.behaviour).toBeDefined(); expect(scatteredSelector.behaviour).toBeDefined(); }); /* ── Coherent fixture details ── */ describe("Coherent fixture assessment", () => { it("phase: cannot_determine (0 resolved — deepening requires resolved >= 3)", () => { console.log(`\n=== Exp 46 Coherent — Phase ===`); console.log(` value: ${coherentResult.phase.value}`); console.log(` confidence: ${coherentResult.phase.confidence}`); console.log(` signals:`, coherentResult.phase.signals); expect(coherentResult.phase.value).toBe("cannot_determine"); }); it("progress: cannot_determine (0 resolved — ratio is null)", () => { console.log(`\n=== Exp 46 Coherent — Progress ===`); console.log(` value: ${coherentResult.progress.value}`); console.log(` confidence: ${coherentResult.progress.confidence}`); expect(coherentResult.progress.value).toBe("cannot_determine"); }); it("conversation health: too_broad", () => { console.log(`\n=== Exp 46 Coherent — Health ===`); console.log(` value: ${coherentResult.conversationHealth.value}`); console.log(` confidence: ${coherentResult.conversationHealth.confidence}`); expect(coherentResult.conversationHealth.value).toBe("too_broad"); }); it("health evidence shows activeUnknownCount = 4", () => { expect(coherentResult.conversationHealth.evidence.activeUnknownCount).toBe(4); expect(coherentResult.conversationHealth.evidence.resolvedNodeRatio).toBe(null); }); it("Clarify is eligible (Rule A: too_broad)", () => { console.log(`\n=== Exp 46 Coherent — Clarify ===`); console.log(` eligible: ${coherentClarEligible}`); expect(coherentClarEligible).toBe(true); }); it("selector produces clarify", () => { console.log(`\n=== Exp 46 Coherent — Selector ===`); console.log(` behaviour: ${coherentSelector.behaviour}`); console.log(` confidence: ${coherentSelector.confidence}`); expect(coherentSelector.behaviour).toBe("clarify"); }); it("understanding trajectory and uncertainty trend are not in current contract", () => { expect(coherentResult.phase).not.toHaveProperty("understandingTrajectory"); expect(coherentResult.phase).not.toHaveProperty("uncertaintyTrend"); }); }); /* ── Scattered fixture details ── */ describe("Scattered fixture assessment", () => { it("phase: cannot_determine (same structural conditions as coherent)", () => { console.log(`\n=== Exp 46 Scattered — Phase ===`); console.log(` value: ${scatteredResult.phase.value}`); console.log(` confidence: ${scatteredResult.phase.confidence}`); expect(scatteredResult.phase.value).toBe("cannot_determine"); }); it("progress: cannot_determine (same structural conditions as coherent)", () => { console.log(`\n=== Exp 46 Scattered — Progress ===`); console.log(` value: ${scatteredResult.progress.value}`); console.log(` confidence: ${scatteredResult.progress.confidence}`); expect(scatteredResult.progress.value).toBe("cannot_determine"); }); it("conversation health: too_broad", () => { console.log(`\n=== Exp 46 Scattered — Health ===`); console.log(` value: ${scatteredResult.conversationHealth.value}`); console.log(` confidence: ${scatteredResult.conversationHealth.confidence}`); expect(scatteredResult.conversationHealth.value).toBe("too_broad"); }); it("health evidence shows activeUnknownCount = 4", () => { expect(scatteredResult.conversationHealth.evidence.activeUnknownCount).toBe(4); expect(scatteredResult.conversationHealth.evidence.resolvedNodeRatio).toBe(null); }); it("Clarify is eligible (Rule A: too_broad)", () => { console.log(`\n=== Exp 46 Scattered — Clarify ===`); console.log(` eligible: ${scatteredClarEligible}`); expect(scatteredClarEligible).toBe(true); }); it("selector produces clarify", () => { console.log(`\n=== Exp 46 Scattered — Selector ===`); console.log(` behaviour: ${scatteredSelector.behaviour}`); console.log(` confidence: ${scatteredSelector.confidence}`); expect(scatteredSelector.behaviour).toBe("clarify"); }); it("understanding trajectory and uncertainty trend are not in current contract", () => { expect(scatteredResult.phase).not.toHaveProperty("understandingTrajectory"); expect(scatteredResult.phase).not.toHaveProperty("uncertaintyTrend"); }); }); /* ── Key discrimination tests ── */ describe("Coherence discrimination", () => { it("both fixtures return too_broad", () => { expect(coherentResult.conversationHealth.value).toBe(scatteredResult.conversationHealth.value); expect(coherentResult.conversationHealth.value).toBe("too_broad"); }); it("Clarify becomes eligible in both", () => { expect(coherentClarEligible).toBe(scatteredClarEligible); expect(coherentClarEligible).toBe(true); }); it("assessor does not distinguish coherent from scattered anywhere (all fields identical)", () => { const cCopy = JSON.parse(JSON.stringify(coherentResult)); const sCopy = JSON.parse(JSON.stringify(scatteredResult)); delete cCopy.assessedAt; delete sCopy.assessedAt; expect(JSON.stringify(cCopy)).toBe(JSON.stringify(sCopy)); }); it("dependency or relationship fields do not influence health result", () => { // Both fixtures have zero edges; the situationGraph edges array is empty expect(coherentFixtureNodes[0].dependsOn.length).toBe(0); expect(scatteredFixtureNodes[0].dependsOn.length).toBe(0); // Assessor's too_broad rule at line 450: activeUnknownCount > 3 && resolved < 2 // Does not reference edges, dependsOn, affects, childIds, or any relationship field }); it("active-unknown count alone determines too_broad in both", () => { expect(coherentResult.conversationHealth.evidence.activeUnknownCount).toBe(4); expect(scatteredResult.conversationHealth.evidence.activeUnknownCount).toBe(4); }); }); /* ── Human-sense review ── */ describe("Human-sense review", () => { it("coherent: too_broad is questionable for a well-structured investigation", () => { console.log(`\n=== Human-sense ===`); console.log(` Coherent health: ${coherentResult.conversationHealth.value}`); console.log(` Four unknowns contributing to one decision — this is breadth, not confusion.`); }); it("scattered: too_broad is believable for genuinely scattered unknowns", () => { console.log(` Scattered health: ${scatteredResult.conversationHealth.value}`); console.log(` Four unrelated threads — this matches the plain-English meaning of "too broad".`); }); }); /* ── Interpretation ── */ describe("Interpretation", () => { it("coherent fixture classified as questionable", () => { expect(coherentResult.conversationHealth.value).toBe("too_broad"); }); it("scattered fixture classified as believable", () => { expect(scatteredResult.conversationHealth.value).toBe("too_broad"); }); it("overall: count is useful but cannot distinguish coherence", () => { expect(coherentResult.conversationHealth.value).toBe(scatteredResult.conversationHealth.value); }); }); /* ── Limitations ── */ describe("Limitations", () => { it("cannot determine what real users judge coherent vs scattered", () => {}); it("does not test graph edge or dependency field effects", () => {}); it("synthetic labels may not capture domain nuance", () => {}); }); /* ── Determinism and immutability ── */ describe("Determinism and immutability", () => { it("assessor does not mutate inputs", () => { const cNodes = JSON.parse(JSON.stringify(coherentFixtureNodes)); const sNodes = JSON.parse(JSON.stringify(scatteredFixtureNodes)); const cInput = buildInput(cNodes, [], cNodes[1].id); const sInput = buildInput(sNodes, [], sNodes[1].id); const cSnap = JSON.stringify(cInput); const sSnap = JSON.stringify(sInput); assessInvestigationState(cInput); assessInvestigationState(sInput); expect(JSON.stringify(cInput)).toBe(cSnap); expect(JSON.stringify(sInput)).toBe(sSnap); }); it("repeated runs are deterministic", () => { const c1 = assessInvestigationState(buildInput(coherentFixtureNodes, [], coherentFixtureNodes[1].id)); const c2 = assessInvestigationState(buildInput(coherentFixtureNodes, [], coherentFixtureNodes[1].id)); const c1C = JSON.parse(JSON.stringify(c1)); const c2C = JSON.parse(JSON.stringify(c2)); delete c1C.assessedAt; delete c2C.assessedAt; expect(JSON.stringify(c1C)).toBe(JSON.stringify(c2C)); const s1 = assessInvestigationState(buildInput(scatteredFixtureNodes, [], scatteredFixtureNodes[1].id)); const s2 = assessInvestigationState(buildInput(scatteredFixtureNodes, [], scatteredFixtureNodes[1].id)); const s1C = JSON.parse(JSON.stringify(s1)); const s2C = JSON.parse(JSON.stringify(s2)); delete s1C.assessedAt; delete s2C.assessedAt; expect(JSON.stringify(s1C)).toBe(JSON.stringify(s2C)); }); it("fixtures remain test-only", () => { expect(true).toBe(true); }); it("production assessor and selector remain unchanged", () => { expect(true).toBe(true); }); }); }); });