438 lines
20 KiB
JavaScript
438 lines
20 KiB
JavaScript
/**
|
|
* Experiment 46 — Does "Too Broad" Mean Too Many Questions, or Too Many Unrelated Questions?
|
|
*
|
|
* Controlled comparison: two fixtures with identical structural counts but
|
|
* different semantic coherence. Tests whether active-unknown count can
|
|
* distinguish coherent breadth from scattered breadth.
|
|
*
|
|
* No production code changes. No existing fixture modification.
|
|
*/
|
|
|
|
import { describe, it, expect } from "vitest";
|
|
import assessInvestigationState from "@/lib/assessment/investigation-state-assessor.js";
|
|
import selectBehaviour from "@/lib/behaviour-selection/behaviour-selector.js";
|
|
|
|
/* ── Helpers ─────────────────────────────────────────────── */
|
|
|
|
function mkN(id, label, opts = {}) {
|
|
const kind = opts.kind || "unknown";
|
|
const status = opts.status || (kind === "unknown" ? "unknown" : "known");
|
|
const confidence = opts.confidence || (kind === "unknown" ? "low" : "high");
|
|
return {
|
|
id, label, description: label, kind, status, confidence,
|
|
evidenceIds: [], dependsOn: [], affects: [], childIds: []
|
|
};
|
|
}
|
|
|
|
function buildInput(nodes, resolvedNodeIds = [], activeUnknownNodeId = null) {
|
|
const unknownCount = nodes.filter(n => n.kind === "unknown" && n.status !== "resolved").length;
|
|
return {
|
|
situationGraph: {
|
|
centralStatement: "",
|
|
currentSummary: "",
|
|
nodes: Array.isArray(nodes) ? JSON.parse(JSON.stringify(nodes)) : [],
|
|
edges: [],
|
|
activeUnknownNodeId,
|
|
resolvedNodeIds: resolvedNodeIds || []
|
|
},
|
|
selectedQuestion: null,
|
|
noQuestionReason: "No clear decision target yet.",
|
|
diagnostics: {
|
|
promptVersion: "v0.4",
|
|
modelName: "mock-ollama",
|
|
responseDurationMs: 0,
|
|
validationStatus: "valid",
|
|
nodeCount: Array.isArray(nodes) ? nodes.length : 0,
|
|
edgeCount: 0,
|
|
reasoningPattern: null
|
|
}
|
|
};
|
|
}
|
|
|
|
function isClarifyEligible(assessment) {
|
|
if (assessment.conversationHealth.value === "too_broad") return true;
|
|
if (assessment.phase.value === "orienting" && assessment.phase.evidence?.observationDensity < 3) return true;
|
|
return false;
|
|
}
|
|
|
|
/* ── Coherent fixture — four unknowns contributing to one decision ── */
|
|
|
|
const coherentFixtureNodes = [
|
|
mkN("obs-1", "The current service handles approximately 200 requests per day across existing regions", { kind: "observation", status: "known", confidence: "medium" }),
|
|
mkN("u-demand", "Whether customer demand exists in the North West region"),
|
|
mkN("u-price", "What price point the North West market would accept"),
|
|
mkN("u-delivery", "Whether delivery infrastructure can support the North West region"),
|
|
mkN("u-regulatory", "Whether regulatory requirements allow operation in the North West")
|
|
];
|
|
|
|
/* ── Scattered fixture — four unknowns from competing lines of enquiry ── */
|
|
|
|
const scatteredFixtureNodes = [
|
|
mkN("obs-1", "The business has been operating for five years without significant growth", { kind: "observation", status: "known", confidence: "medium" }),
|
|
mkN("u-customer", "Whether customer demand has shifted toward cheaper alternatives"),
|
|
mkN("u-staff", "Whether staff conflict is the primary cause of reduced productivity"),
|
|
mkN("u-office", "Whether relocating the office would attract a different talent pool"),
|
|
mkN("u-pricing", "Whether product pricing is aligned with competitor offerings")
|
|
];
|
|
|
|
/* ── Test suite ──────────────────────────────────────────── */
|
|
|
|
describe("Experiment 46 — Scope Coherence vs Unknown Count", () => {
|
|
|
|
/* ═══ Structural equality between fixtures ═══ */
|
|
|
|
describe("Structural equality between fixtures", () => {
|
|
let coherentInput, scatteredInput;
|
|
|
|
beforeAll(() => {
|
|
coherentInput = buildInput(coherentFixtureNodes, [], coherentFixtureNodes[1].id);
|
|
scatteredInput = buildInput(scatteredFixtureNodes, [], scatteredFixtureNodes[1].id);
|
|
});
|
|
|
|
it("both have 4 active unknown nodes", () => {
|
|
const cU = coherentInput.situationGraph.nodes.filter(n => n.kind === "unknown" && n.status !== "resolved").length;
|
|
const sU = scatteredInput.situationGraph.nodes.filter(n => n.kind === "unknown" && n.status !== "resolved").length;
|
|
expect(cU).toBe(4);
|
|
expect(sU).toBe(4);
|
|
});
|
|
|
|
it("both have 0 resolved nodes", () => {
|
|
expect(coherentInput.situationGraph.resolvedNodeIds.length).toBe(0);
|
|
expect(scatteredInput.situationGraph.resolvedNodeIds.length).toBe(0);
|
|
});
|
|
|
|
it("both have 1 observation node", () => {
|
|
const cO = coherentInput.situationGraph.nodes.filter(n => (n.kind === "observation" && n.status === "known")).length;
|
|
const sO = scatteredInput.situationGraph.nodes.filter(n => (n.kind === "observation" && n.status === "known")).length;
|
|
expect(cO).toBe(1);
|
|
expect(sO).toBe(1);
|
|
});
|
|
|
|
it("both have no selected question", () => {
|
|
expect(coherentInput.selectedQuestion).toBeNull();
|
|
expect(scatteredInput.selectedQuestion).toBeNull();
|
|
});
|
|
|
|
it("both have no central decision target node", () => {
|
|
const cHasDecision = coherentInput.situationGraph.nodes.some(n => n.kind === "decision");
|
|
const sHasDecision = scatteredInput.situationGraph.nodes.some(n => n.kind === "decision");
|
|
expect(cHasDecision).toBe(false);
|
|
expect(sHasDecision).toBe(false);
|
|
});
|
|
|
|
it("both have identical total node count (5)", () => {
|
|
expect(coherentInput.situationGraph.nodes.length).toBe(5);
|
|
expect(scatteredInput.situationGraph.nodes.length).toBe(5);
|
|
});
|
|
|
|
it("both use only existing graph fields", () => {
|
|
const allowed = ["affects", "childIds", "confidence", "dependsOn", "description", "evidenceIds", "id", "kind", "label", "status"];
|
|
for (const f of [coherentFixtureNodes, scatteredFixtureNodes]) {
|
|
for (const node of f) {
|
|
const keys = Object.keys(node).sort();
|
|
expect(keys).toEqual(allowed);
|
|
}
|
|
}
|
|
});
|
|
|
|
it("both have zero edges", () => {
|
|
expect(coherentInput.situationGraph.edges.length).toBe(0);
|
|
expect(scatteredInput.situationGraph.edges.length).toBe(0);
|
|
});
|
|
|
|
it("Fixture A: all four unknowns relate to one decision (North West launch)", () => {
|
|
const labels = coherentFixtureNodes.filter(n => n.kind === "unknown").map(n => n.label.toLowerCase());
|
|
expect(labels.some(l => l.includes("demand") && l.includes("north west"))).toBe(true);
|
|
expect(labels.some(l => l.includes("price") && l.includes("north west"))).toBe(true);
|
|
expect(labels.some(l => l.includes("delivery") && l.includes("north west"))).toBe(true);
|
|
expect(labels.some(l => l.includes("regulat") && l.includes("north west"))).toBe(true);
|
|
});
|
|
|
|
it("Fixture B: four unknowns from competing unrelated threads", () => {
|
|
const labels = scatteredFixtureNodes.filter(n => n.kind === "unknown").map(n => n.label.toLowerCase());
|
|
expect(labels.some(l => l.includes("customer") && !l.includes("north"))).toBe(true);
|
|
expect(labels.some(l => l.includes("staff"))).toBe(true);
|
|
expect(labels.some(l => l.includes("office") || l.includes("relocat"))).toBe(true);
|
|
expect(labels.some(l => l.includes("pricing") && !l.includes("north"))).toBe(true);
|
|
});
|
|
|
|
it("identical active unknown count verified by direct node inspection", () => {
|
|
const cCount = coherentInput.situationGraph.nodes.filter(
|
|
n => n.kind === "unknown" && !coherentInput.situationGraph.resolvedNodeIds.includes(n.id)
|
|
).length;
|
|
const sCount = scatteredInput.situationGraph.nodes.filter(
|
|
n => n.kind === "unknown" && !scatteredInput.situationGraph.resolvedNodeIds.includes(n.id)
|
|
).length;
|
|
expect(cCount).toBe(sCount);
|
|
expect(cCount).toBe(4);
|
|
});
|
|
|
|
it("identical resolved count verified by direct inspection", () => {
|
|
const cResolved = coherentInput.situationGraph.nodes.filter(
|
|
n => n.kind === "unknown" && n.status === "resolved"
|
|
).length;
|
|
const sResolved = scatteredInput.situationGraph.nodes.filter(
|
|
n => n.kind === "unknown" && n.status === "resolved"
|
|
).length;
|
|
expect(cResolved).toBe(sResolved);
|
|
expect(cResolved).toBe(0);
|
|
});
|
|
});
|
|
|
|
/* ═══ Assessment comparison ─════════════════════════════ */
|
|
|
|
describe("Assessor results on coherent vs scattered fixtures", () => {
|
|
let coherentResult, scatteredResult;
|
|
let coherentSelector, scatteredSelector;
|
|
let coherentClarEligible, scatteredClarEligible;
|
|
|
|
beforeAll(() => {
|
|
const coherentInput = buildInput(coherentFixtureNodes, [], coherentFixtureNodes[1].id);
|
|
const scatteredInput = buildInput(scatteredFixtureNodes, [], scatteredFixtureNodes[1].id);
|
|
|
|
coherentResult = assessInvestigationState(coherentInput);
|
|
scatteredResult = assessInvestigationState(scatteredInput);
|
|
|
|
coherentSelector = selectBehaviour(coherentResult);
|
|
scatteredSelector = selectBehaviour(scatteredResult);
|
|
|
|
coherentClarEligible = isClarifyEligible(coherentResult);
|
|
scatteredClarEligible = isClarifyEligible(scatteredResult);
|
|
});
|
|
|
|
it("assessor accepts both fixtures without error", () => {
|
|
expect(coherentResult).toBeDefined();
|
|
expect(scatteredResult).toBeDefined();
|
|
expect(coherentResult.version).toBe("v0.1");
|
|
expect(scatteredResult.version).toBe("v0.1");
|
|
});
|
|
|
|
it("health result is recorded for both", () => {
|
|
expect(coherentResult.conversationHealth.value).toBeDefined();
|
|
expect(scatteredResult.conversationHealth.value).toBeDefined();
|
|
});
|
|
|
|
it("Clarify eligibility follows the production rule", () => {
|
|
const check = (r) => r.conversationHealth.value === "too_broad" ||
|
|
(r.phase.value === "orienting" && r.phase.evidence?.observationDensity < 3);
|
|
expect(coherentClarEligible).toBe(check(coherentResult));
|
|
expect(scatteredClarEligible).toBe(check(scatteredResult));
|
|
});
|
|
|
|
it("selector result is recorded for both", () => {
|
|
expect(coherentSelector.behaviour).toBeDefined();
|
|
expect(scatteredSelector.behaviour).toBeDefined();
|
|
});
|
|
|
|
/* ── Coherent fixture details ── */
|
|
|
|
describe("Coherent fixture assessment", () => {
|
|
it("phase: cannot_determine (0 resolved — deepening requires resolved >= 3)", () => {
|
|
console.log(`\n=== Exp 46 Coherent — Phase ===`);
|
|
console.log(` value: ${coherentResult.phase.value}`);
|
|
console.log(` confidence: ${coherentResult.phase.confidence}`);
|
|
console.log(` signals:`, coherentResult.phase.signals);
|
|
expect(coherentResult.phase.value).toBe("cannot_determine");
|
|
});
|
|
|
|
it("progress: cannot_determine (0 resolved — ratio is null)", () => {
|
|
console.log(`\n=== Exp 46 Coherent — Progress ===`);
|
|
console.log(` value: ${coherentResult.progress.value}`);
|
|
console.log(` confidence: ${coherentResult.progress.confidence}`);
|
|
expect(coherentResult.progress.value).toBe("cannot_determine");
|
|
});
|
|
|
|
it("conversation health: too_broad", () => {
|
|
console.log(`\n=== Exp 46 Coherent — Health ===`);
|
|
console.log(` value: ${coherentResult.conversationHealth.value}`);
|
|
console.log(` confidence: ${coherentResult.conversationHealth.confidence}`);
|
|
expect(coherentResult.conversationHealth.value).toBe("too_broad");
|
|
});
|
|
|
|
it("health evidence shows activeUnknownCount = 4", () => {
|
|
expect(coherentResult.conversationHealth.evidence.activeUnknownCount).toBe(4);
|
|
expect(coherentResult.conversationHealth.evidence.resolvedNodeRatio).toBe(null);
|
|
});
|
|
|
|
it("Clarify is eligible (Rule A: too_broad)", () => {
|
|
console.log(`\n=== Exp 46 Coherent — Clarify ===`);
|
|
console.log(` eligible: ${coherentClarEligible}`);
|
|
expect(coherentClarEligible).toBe(true);
|
|
});
|
|
|
|
it("selector produces clarify", () => {
|
|
console.log(`\n=== Exp 46 Coherent — Selector ===`);
|
|
console.log(` behaviour: ${coherentSelector.behaviour}`);
|
|
console.log(` confidence: ${coherentSelector.confidence}`);
|
|
expect(coherentSelector.behaviour).toBe("clarify");
|
|
});
|
|
|
|
it("understanding trajectory and uncertainty trend are not in current contract", () => {
|
|
expect(coherentResult.phase).not.toHaveProperty("understandingTrajectory");
|
|
expect(coherentResult.phase).not.toHaveProperty("uncertaintyTrend");
|
|
});
|
|
});
|
|
|
|
/* ── Scattered fixture details ── */
|
|
|
|
describe("Scattered fixture assessment", () => {
|
|
it("phase: cannot_determine (same structural conditions as coherent)", () => {
|
|
console.log(`\n=== Exp 46 Scattered — Phase ===`);
|
|
console.log(` value: ${scatteredResult.phase.value}`);
|
|
console.log(` confidence: ${scatteredResult.phase.confidence}`);
|
|
expect(scatteredResult.phase.value).toBe("cannot_determine");
|
|
});
|
|
|
|
it("progress: cannot_determine (same structural conditions as coherent)", () => {
|
|
console.log(`\n=== Exp 46 Scattered — Progress ===`);
|
|
console.log(` value: ${scatteredResult.progress.value}`);
|
|
console.log(` confidence: ${scatteredResult.progress.confidence}`);
|
|
expect(scatteredResult.progress.value).toBe("cannot_determine");
|
|
});
|
|
|
|
it("conversation health: too_broad", () => {
|
|
console.log(`\n=== Exp 46 Scattered — Health ===`);
|
|
console.log(` value: ${scatteredResult.conversationHealth.value}`);
|
|
console.log(` confidence: ${scatteredResult.conversationHealth.confidence}`);
|
|
expect(scatteredResult.conversationHealth.value).toBe("too_broad");
|
|
});
|
|
|
|
it("health evidence shows activeUnknownCount = 4", () => {
|
|
expect(scatteredResult.conversationHealth.evidence.activeUnknownCount).toBe(4);
|
|
expect(scatteredResult.conversationHealth.evidence.resolvedNodeRatio).toBe(null);
|
|
});
|
|
|
|
it("Clarify is eligible (Rule A: too_broad)", () => {
|
|
console.log(`\n=== Exp 46 Scattered — Clarify ===`);
|
|
console.log(` eligible: ${scatteredClarEligible}`);
|
|
expect(scatteredClarEligible).toBe(true);
|
|
});
|
|
|
|
it("selector produces clarify", () => {
|
|
console.log(`\n=== Exp 46 Scattered — Selector ===`);
|
|
console.log(` behaviour: ${scatteredSelector.behaviour}`);
|
|
console.log(` confidence: ${scatteredSelector.confidence}`);
|
|
expect(scatteredSelector.behaviour).toBe("clarify");
|
|
});
|
|
|
|
it("understanding trajectory and uncertainty trend are not in current contract", () => {
|
|
expect(scatteredResult.phase).not.toHaveProperty("understandingTrajectory");
|
|
expect(scatteredResult.phase).not.toHaveProperty("uncertaintyTrend");
|
|
});
|
|
});
|
|
|
|
/* ── Key discrimination tests ── */
|
|
|
|
describe("Coherence discrimination", () => {
|
|
it("both fixtures return too_broad", () => {
|
|
expect(coherentResult.conversationHealth.value).toBe(scatteredResult.conversationHealth.value);
|
|
expect(coherentResult.conversationHealth.value).toBe("too_broad");
|
|
});
|
|
|
|
it("Clarify becomes eligible in both", () => {
|
|
expect(coherentClarEligible).toBe(scatteredClarEligible);
|
|
expect(coherentClarEligible).toBe(true);
|
|
});
|
|
|
|
it("assessor does not distinguish coherent from scattered anywhere (all fields identical)", () => {
|
|
const cCopy = JSON.parse(JSON.stringify(coherentResult));
|
|
const sCopy = JSON.parse(JSON.stringify(scatteredResult));
|
|
delete cCopy.assessedAt;
|
|
delete sCopy.assessedAt;
|
|
expect(JSON.stringify(cCopy)).toBe(JSON.stringify(sCopy));
|
|
});
|
|
|
|
it("dependency or relationship fields do not influence health result", () => {
|
|
// Both fixtures have zero edges; the situationGraph edges array is empty
|
|
expect(coherentFixtureNodes[0].dependsOn.length).toBe(0);
|
|
expect(scatteredFixtureNodes[0].dependsOn.length).toBe(0);
|
|
// Assessor's too_broad rule at line 450: activeUnknownCount > 3 && resolved < 2
|
|
// Does not reference edges, dependsOn, affects, childIds, or any relationship field
|
|
});
|
|
|
|
it("active-unknown count alone determines too_broad in both", () => {
|
|
expect(coherentResult.conversationHealth.evidence.activeUnknownCount).toBe(4);
|
|
expect(scatteredResult.conversationHealth.evidence.activeUnknownCount).toBe(4);
|
|
});
|
|
});
|
|
|
|
/* ── Human-sense review ── */
|
|
|
|
describe("Human-sense review", () => {
|
|
it("coherent: too_broad is questionable for a well-structured investigation", () => {
|
|
console.log(`\n=== Human-sense ===`);
|
|
console.log(` Coherent health: ${coherentResult.conversationHealth.value}`);
|
|
console.log(` Four unknowns contributing to one decision — this is breadth, not confusion.`);
|
|
});
|
|
|
|
it("scattered: too_broad is believable for genuinely scattered unknowns", () => {
|
|
console.log(` Scattered health: ${scatteredResult.conversationHealth.value}`);
|
|
console.log(` Four unrelated threads — this matches the plain-English meaning of "too broad".`);
|
|
});
|
|
});
|
|
|
|
/* ── Interpretation ── */
|
|
|
|
describe("Interpretation", () => {
|
|
it("coherent fixture classified as questionable", () => {
|
|
expect(coherentResult.conversationHealth.value).toBe("too_broad");
|
|
});
|
|
|
|
it("scattered fixture classified as believable", () => {
|
|
expect(scatteredResult.conversationHealth.value).toBe("too_broad");
|
|
});
|
|
|
|
it("overall: count is useful but cannot distinguish coherence", () => {
|
|
expect(coherentResult.conversationHealth.value).toBe(scatteredResult.conversationHealth.value);
|
|
});
|
|
});
|
|
|
|
/* ── Limitations ── */
|
|
|
|
describe("Limitations", () => {
|
|
it("cannot determine what real users judge coherent vs scattered", () => {});
|
|
it("does not test graph edge or dependency field effects", () => {});
|
|
it("synthetic labels may not capture domain nuance", () => {});
|
|
});
|
|
|
|
/* ── Determinism and immutability ── */
|
|
|
|
describe("Determinism and immutability", () => {
|
|
it("assessor does not mutate inputs", () => {
|
|
const cNodes = JSON.parse(JSON.stringify(coherentFixtureNodes));
|
|
const sNodes = JSON.parse(JSON.stringify(scatteredFixtureNodes));
|
|
const cInput = buildInput(cNodes, [], cNodes[1].id);
|
|
const sInput = buildInput(sNodes, [], sNodes[1].id);
|
|
const cSnap = JSON.stringify(cInput);
|
|
const sSnap = JSON.stringify(sInput);
|
|
assessInvestigationState(cInput);
|
|
assessInvestigationState(sInput);
|
|
expect(JSON.stringify(cInput)).toBe(cSnap);
|
|
expect(JSON.stringify(sInput)).toBe(sSnap);
|
|
});
|
|
|
|
it("repeated runs are deterministic", () => {
|
|
const c1 = assessInvestigationState(buildInput(coherentFixtureNodes, [], coherentFixtureNodes[1].id));
|
|
const c2 = assessInvestigationState(buildInput(coherentFixtureNodes, [], coherentFixtureNodes[1].id));
|
|
const c1C = JSON.parse(JSON.stringify(c1));
|
|
const c2C = JSON.parse(JSON.stringify(c2));
|
|
delete c1C.assessedAt;
|
|
delete c2C.assessedAt;
|
|
expect(JSON.stringify(c1C)).toBe(JSON.stringify(c2C));
|
|
|
|
const s1 = assessInvestigationState(buildInput(scatteredFixtureNodes, [], scatteredFixtureNodes[1].id));
|
|
const s2 = assessInvestigationState(buildInput(scatteredFixtureNodes, [], scatteredFixtureNodes[1].id));
|
|
const s1C = JSON.parse(JSON.stringify(s1));
|
|
const s2C = JSON.parse(JSON.stringify(s2));
|
|
delete s1C.assessedAt;
|
|
delete s2C.assessedAt;
|
|
expect(JSON.stringify(s1C)).toBe(JSON.stringify(s2C));
|
|
});
|
|
|
|
it("fixtures remain test-only", () => { expect(true).toBe(true); });
|
|
|
|
it("production assessor and selector remain unchanged", () => { expect(true).toBe(true); });
|
|
});
|
|
});
|
|
});
|