/** * Experiment 22 — Tests for assessQuestionAgainstDecisionConditions. * * Validates the new classifier against explicit decision conditions. */ import { describe, expect, it } from "vitest"; import { assessQuestionAgainstDecisionConditions } from "@/lib/graph/question-decision-conditions.js"; import { assessQuestionRelevanceToDecision } from "@/lib/graph/question-decision-relevance.js"; import { buildScenarioFixture } from "@/lib/mocks/scenarios.js"; /* ── Helpers ─────────────────────────────────────────────── */ const DECISION_TARGET = "Should we enter the European market with our SaaS analytics platform?"; const CONDITIONS = [ "Credible customer demand exists in Europe", "European compliance is achievable", "The expected market value justifies the cost of entry", "The product offers sufficient competitive differentiation", ]; function makeUnknown(id, label) { return { id, label: label || `Unknown ${id}`, description: label || `Unknown ${id}`, kind: "unknown", status: "unknown", confidence: "low", value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], childIds: [], }; } function makeGraphFor(nodes) { return { nodes, edges: [], resolvedNodeIds: [], activeUnknownNodeId: null, centralStatement: "Test", currentSummary: "Test", reasoningState: null, }; } /* ── tests_deciding_condition ─────────────────────────────── */ describe("assessQuestionAgainstDecisionConditions — tests_deciding_condition", () => { it("classifies a demand question as tests_deciding_condition", () => { const result = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: makeUnknown("d1", "Whether there is genuine demand for our category in Europe"), }); expect(result.relevance).toBe("tests_deciding_condition"); expect(result.matchedCondition).toContain("demand"); }); it("classifies a compliance question as tests_deciding_condition", () => { const result = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: makeUnknown("d2", "Whether our product is suitable for European compliance requirements"), }); expect(result.relevance).toBe("tests_deciding_condition"); expect(result.matchedCondition).toContain("compliance"); }); it("classifies a competitive differentiation question as tests_deciding_condition", () => { const result = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: makeUnknown("d3", "Whether we have competitive differentiation against existing European players"), }); expect(result.relevance).toBe("tests_deciding_condition"); expect(result.matchedCondition).toContain("differentiation"); }); it("classifies a value justification question as tests_deciding_condition", () => { const result = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: makeUnknown("d4", "Whether the expected market value justifies the compliance investment"), }); expect(result.relevance).toBe("tests_deciding_condition"); expect(result.matchedCondition).toContain("value"); }); }); /* ── adds_supporting_evidence ────────────────────────────── */ describe("assessQuestionAgainstDecisionConditions — adds_supporting_evidence", () => { it("classifies a customer segment question as adds_supporting_evidence", () => { const result = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: makeUnknown("s1", "Which European countries should we target first"), }); expect(result.relevance).toBe("adds_supporting_evidence"); }); it("classifies an investment detail question as adds_supporting_evidence", () => { const result = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: makeUnknown("s2", "How should we structure the European subsidiary"), }); expect(result.relevance).toBe("adds_supporting_evidence"); }); }); /* ── outside_decision_conditions ─────────────────────────── */ describe("assessQuestionAgainstDecisionConditions — outside_decision_conditions", () => { it("classifies a background question as outside_decision_conditions", () => { const result = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: makeUnknown("o1", "What is the company's current US headquarters location"), }); expect(result.relevance).toBe("outside_decision_conditions"); }); it("classifies an unrelated observation as outside_decision_conditions", () => { const result = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: makeUnknown("o2", "Our CEO's previous experience in Asian markets"), }); expect(result.relevance).toBe("outside_decision_conditions"); }); }); /* ── Multiple condition matches ──────────────────────────── */ describe("assessQuestionAgainstDecisionConditions — multiple conditions matched", () => { it("returns the first matching condition when a question spans two conditions", () => { const result = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: makeUnknown("m1", "What market demand and compliance requirements exist in Europe"), }); expect(result.relevance).toBe("tests_deciding_condition"); // First condition that matches should be returned expect(result.matchedCondition).toContain("demand"); }); it("treats tests_deciding_condition as higher priority than adds_supporting_evidence", () => { const result = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: makeUnknown("m2", "Does our platform meet European data regulations"), }); expect(result.relevance).toBe("tests_deciding_condition"); }); }); /* ── cannot_determine — missing input ────────────────────── */ describe("assessQuestionAgainstDecisionConditions — cannot_determine (missing input)", () => { it("returns cannot_determine when decisionTarget is missing", () => { const result = assessQuestionAgainstDecisionConditions({ decisionConditions: CONDITIONS, unknown: makeUnknown("mi1", "Whether to enter Europe"), }); expect(result.relevance).toBe("cannot_determine"); }); it("returns cannot_determine when decisionTarget is empty string", () => { const result = assessQuestionAgainstDecisionConditions({ decisionTarget: "", decisionConditions: CONDITIONS, unknown: makeUnknown("mi2", "Whether to enter Europe"), }); expect(result.relevance).toBe("cannot_determine"); }); it("returns cannot_determine when decisionTarget is too short", () => { const result = assessQuestionAgainstDecisionConditions({ decisionTarget: "Europe", decisionConditions: CONDITIONS, unknown: makeUnknown("mi3", "Whether to enter"), }); expect(result.relevance).toBe("cannot_determine"); }); it("returns cannot_determine when unknown is missing", () => { const result = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, }); expect(result.relevance).toBe("cannot_determine"); }); it("returns cannot_determine when unknown has empty label and description", () => { const node = makeUnknown("mi4", ""); node.label = ""; node.description = ""; const result = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: node, }); expect(result.relevance).toBe("cannot_determine"); }); it("returns cannot_determine when no input object is provided", () => { const result = assessQuestionAgainstDecisionConditions(null); expect(result.relevance).toBe("cannot_determine"); }); it("returns cannot_determine when unknown node has no kind property", () => { const node = makeUnknown("mi5", "test"); delete node.kind; const result = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: node, }); expect(result.relevance).toBe("cannot_determine"); }); }); /* ── cannot_determine — missing / empty conditions ──────── */ describe("assessQuestionAgainstDecisionConditions — cannot_determine (missing conditions)", () => { it("returns cannot_determine when decisionConditions is missing", () => { const result = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, unknown: makeUnknown("mc1", "Whether to enter Europe"), }); expect(result.relevance).toBe("cannot_determine"); }); it("returns cannot_determine when decisionConditions is an empty array", () => { const result = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: [], unknown: makeUnknown("mc2", "Whether to enter Europe"), }); expect(result.relevance).toBe("cannot_determine"); }); it("returns cannot_determine when all conditions are empty strings", () => { const result = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: ["", " ", ""], unknown: makeUnknown("mc3", "Whether to enter Europe"), }); expect(result.relevance).toBe("cannot_determine"); }); }); /* ── Deterministic output ─────────────────────────────────── */ describe("assessQuestionAgainstDecisionConditions — deterministic output", () => { it("returns identical results for identical inputs", () => { const node = makeUnknown("det1", "Whether there is genuine demand in Europe"); const input = { decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: node }; const r1 = assessQuestionAgainstDecisionConditions(input); const r2 = assessQuestionAgainstDecisionConditions(input); expect(r1).toEqual(r2); }); it("returns the same classification across multiple calls", () => { const results = []; for (let i = 0; i < 5; i++) { const result = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: makeUnknown(`det2-${i}`, "Does our platform comply with GDPR"), }); results.push(result.relevance); } expect(new Set(results).size).toBe(1); }); }); /* ── Input immutability ──────────────────────────────────── */ describe("assessQuestionAgainstDecisionConditions — input immutability", () => { it("does not mutate the unknown node", () => { const node = makeUnknown("imm1", "Whether to proceed with European market entry"); const snapshot = JSON.parse(JSON.stringify(node)); assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: node }); expect(node).toEqual(snapshot); }); it("does not mutate the graph (when provided)", () => { const node = makeUnknown("imm2", "test"); const graph = makeGraphFor([node]); const originalNodesJSON = JSON.stringify(graph.nodes); assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: node, graph }); expect(JSON.stringify(graph.nodes)).toBe(originalNodesJSON); }); it("does not mutate the decisionTarget string", () => { const dt = "Should we enter the European market with our SaaS analytics platform?"; assessQuestionAgainstDecisionConditions({ decisionTarget: dt, decisionConditions: CONDITIONS, unknown: makeUnknown("imm3", "test") }); expect(dt).toBe("Should we enter the European market with our SaaS analytics platform?"); }); it("does not mutate the decisionConditions array", () => { const conditions = [...CONDITIONS]; assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: conditions, unknown: makeUnknown("imm4", "test") }); expect(JSON.stringify(conditions)).toBe(JSON.stringify(CONDITIONS)); }); }); /* ── Complete long-investigation sequence ──────────────────── */ describe("assessQuestionAgainstDecisionConditions — long investigation sequence", () => { const scenarioName = "long"; const turnCount = 5; const allResults = []; for (let t = 0; t < turnCount; t++) { const fixture = buildScenarioFixture(scenarioName, t); expect(fixture).not.toBeNull(`Turn ${t} should have a valid fixture`); const decisionTarget = fixture.situationGraph.centralStatement; const resolvedIds = new Set(fixture.situationGraph.resolvedNodeIds || []); for (const node of fixture.situationGraph.nodes) { if (node.kind !== "unknown") continue; if (resolvedIds.has(node.id)) continue; const result = assessQuestionAgainstDecisionConditions({ decisionTarget, decisionConditions: CONDITIONS, unknown: node, graph: fixture.situationGraph, }); allResults.push({ turn: t, nodeId: node.id, label: node.label, relevance: result.relevance, matchedCondition: result.matchedCondition, reason: result.reason }); } } it("every unresolved unknown across all turns receives a valid classification", () => { const validCategories = ["tests_deciding_condition", "adds_supporting_evidence", "outside_decision_conditions", "cannot_determine"]; for (const r of allResults) { expect(validCategories).toContain(r.relevance); } }); it("every unresolved unknown in the long sequence classifies as tests_deciding_condition", () => { for (const r of allResults) { expect(r.relevance).toBe("tests_deciding_condition"); } }); it("each unresolved unknown matches its corresponding decision condition", () => { const expectedMatches = [ { turn: 0, nodeId: "u-1", contains: "demand" }, { turn: 1, nodeId: "u-2", contains: "compliance" }, { turn: 2, nodeId: "u-3", contains: "value" }, { turn: 3, nodeId: "u-4", contains: "differentiation" }, ]; for (const expected of expectedMatches) { const r = allResults.find((x) => x.turn === expected.turn && x.nodeId === expected.nodeId); expect(r).toBeDefined(); expect(r.relevance).toBe("tests_deciding_condition"); expect(r.matchedCondition).toContain(expected.contains); } }); it("the sequence runs deterministically and does not mutate inputs", () => { const node = makeUnknown("det-seq", "Whether there is genuine demand in Europe"); const r1 = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: node }); const r2 = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: node }); expect(r1).toEqual(r2); expect(JSON.stringify(node)).toBe(JSON.stringify(makeUnknown("det-seq", "Whether there is genuine demand in Europe"))); }); it("category distribution across the full sequence", () => { const categories = {}; for (const r of allResults) { categories[r.relevance] = (categories[r.relevance] || 0) + 1; } console.log("\n=== Experiment 22 — Classification Results ==="); console.log(`Total unresolved unknowns classified: ${allResults.length}`); console.log("Category distribution:", JSON.stringify(categories, null, 2)); expect(allResults.length).toBeGreaterThan(0); }); it("detailed per-turn classifications", () => { const byTurn = {}; for (const r of allResults) { if (!byTurn[r.turn]) byTurn[r.turn] = []; byTurn[r.turn].push(r); } for (const [turn, results] of Object.entries(byTurn)) { console.log(`\n--- Turn ${turn} (${results.length} unresolved unknowns) ---`); for (const r of results) { const shortLabel = r.label?.length > 60 ? r.label.slice(0, 60) + "..." : r.label; console.log(` [${r.relevance}] ${shortLabel}`); } } expect(allResults.length).toBeGreaterThan(0); }); it("Turn 0 (market demand) is classified as tests_deciding_condition", () => { const turn0 = allResults.find((r) => r.turn === 0); expect(turn0).toBeDefined(); expect(turn0.relevance).toBe("tests_deciding_condition"); }); it("Turns 1-2 (compliance, cost) are classified as tests_deciding_condition", () => { const turns1to2 = allResults.filter((r) => r.turn >= 1 && r.turn <= 2); for (const r of turns1to2) { expect(r.relevance).toBe("tests_deciding_condition"); } }); it("Turn 3 (differentiation) is classified as tests_deciding_condition", () => { const turn3 = allResults.find((r) => r.turn === 3); expect(turn3).toBeDefined(); expect(turn3.relevance).toBe("tests_deciding_condition"); }); it("competitive differentiation becomes a clearly decisive classification", () => { const differentiating = allResults.find((r) => r.nodeId === "u-4" && r.turn === 3); expect(differentiating).toBeDefined(); expect(differentiating.relevance).toBe("tests_deciding_condition"); expect(differentiating.matchedCondition).toContain("differentiation"); }); it("all classified questions include a non-empty reason", () => { for (const r of allResults) { if (r.relevance !== "cannot_determine") { expect(typeof r.reason).toBe("string"); expect(r.reason.length).toBeGreaterThan(0); } } }); it("every unresolved unknown has a matchedCondition when classified", () => { for (const r of allResults) { if (r.relevance !== "cannot_determine" && r.relevance !== "outside_decision_conditions") { expect(r.matchedCondition).toBeDefined(); expect(typeof r.matchedCondition).toBe("string"); expect(r.matchedCondition.length).toBeGreaterThan(0); } } }); it("the complete long-investigation sequence runs without error and produces results", () => { expect(allResults.length).toBeGreaterThan(0); }); }); /* ── Comparison with Experiment 21 — clearer distinction ─ */ describe("assessQuestionAgainstDecisionConditions — comparison with Experiment 21", () => { it("distinguishes between decisive and supporting questions (Experiment 22 vs 21)", () => { const conditions = CONDITIONS; const decisions21 = []; const decisions22 = []; for (let t = 0; t < 4; t++) { const fixture = buildScenarioFixture("long", t); if (!fixture) continue; const decisionTarget = fixture.situationGraph.centralStatement; for (const node of fixture.situationGraph.nodes) { if (node.kind !== "unknown") continue; const resolvedIds = new Set(fixture.situationGraph.resolvedNodeIds || []); if (resolvedIds.has(node.id)) continue; // Experiment 21 classification const r21 = assessQuestionRelevanceToDecision({ decisionTarget, unknown: node }); decisions21.push({ turn: t, label: node.label, relevance: r21.relevance }); // Experiment 22 classification const r22 = assessQuestionAgainstDecisionConditions({ decisionTarget, decisionConditions: conditions, unknown: node }); decisions22.push({ turn: t, label: node.label, relevance: r22.relevance }); } } const distinct21 = new Set(decisions21.map((r) => r.relevance)); const distinct22 = new Set(decisions22.map((r) => r.relevance)); // Experiment 22 should have a clearer distinction than 21 expect(distinct21.size).toBeGreaterThanOrEqual(1); expect(distinct22.size).toBeGreaterThanOrEqual(1); // Verify at least some questions moved to tests_deciding_condition in Exp 22 const decisiveCount = decisions22.filter((r) => r.relevance === "tests_deciding_condition").length; expect(decisiveCount).toBeGreaterThan(0); }); });