diff --git a/docs/design-evolution-log.md b/docs/design-evolution-log.md index 66be910..b044585 100644 --- a/docs/design-evolution-log.md +++ b/docs/design-evolution-log.md @@ -1001,6 +1001,42 @@ If none of these can be evaluated after 2–3 real investigations with v0.1, the --- +### Experiment 20 — Passive Question Importance Classification + +#### Hypothesis + +Does a passive classifier that tags unresolved unknowns as `important`, `helpful`, `incidental`, or `cannot_determine` (using only existing graph fields, no scoring, no weights) produce coherent importance patterns across normal investigations? + +This is one question. Nothing else matters until this is answered. + +#### Scope + +A pure function `assessQuestionImportance({ node, graph })` implementing three deterministic rules: + +1. **important** — Other unresolved unknown(s) depend on this one (via `dependsOn` or edges); OR text contains decision-context patterns ("whether to", "build", "launch") AND has ≥1 graph connection. +2. **helpful** — Text contains evidence-related patterns ("evidence", "metric", "measure", "criteria"); OR has ≥2 total connections in the graph. +3. **incidental** — Default when neither important nor helpful conditions are met. +4. **cannot_determine** — Node label and description are both empty/null (fallback for empty input). + +The classifier is passive — validated only against mock scenario fixtures. No changes to: graph construction, unknown selection, question selection, prompts, Ollama integration, APIs, UI, state assessment, behaviour selection, or conversation output. + +#### Validation + +Run the classifier passively against existing mock scenarios (comparison, contradictory, missing-evidence, decision, long investigation, complete) and verify at least three classifications align with intuitive expectations: + +- The "decision" scenario's build/commercial unknown → `important` +- An evidence-gathering unknown from the comparison scenario → `helpful` +- A minor formatting or cosmetic unknown → `incidental` + +#### Open Questions + +- Which importance category appears most frequently across normal investigations? +- Does the downstream-dependency rule align with how the engine currently prioritises (score-based selection)? +- Are decision-context text patterns ("whether to", "build") capturing the right signal, or is this too coarse-grained? +- Can a future experiment use these categories to influence question phrasing (not priority) without breaking existing selection? + +--- + ## Phase Transition Record that the project has moved from: diff --git a/lib/graph/question-importance.js b/lib/graph/question-importance.js new file mode 100644 index 0000000..d728174 --- /dev/null +++ b/lib/graph/question-importance.js @@ -0,0 +1,137 @@ +/** + * Question Importance — passive classifier for unresolved unknowns. + * + * A pure-function layer that classifies each unresolved unknown into one of + * four importance categories using only actual repository fields. No scoring, + * no weights, no new graph structure. Designed to be validated against mock + * scenarios without changing engine behaviour in any way. + * + * Classification rules (in order): + * 1. important — Other unresolved unknown(s) depend on this one being resolved first; + * OR text contains decision-context patterns ("whether to", "build", + * "launch", "continue", "proceed") AND has ≥1 graph connection. + * 2. helpful — Text contains evidence-related patterns (evidence, metric, measure, + * criteria, validation, proof); OR has ≥2 total connections in the graph. + * 3. incidental — Default when neither important nor helpful conditions are met. + * 4. cannot_determine — Node label and description are both empty/null. + * + * IMPORTANT: This module does NOT modify engine behaviour. It must never write to + * the graph, change unknown selection, or influence question generation. Validation + * is done by running this classifier passively against existing scenario fixtures. + */ + +/* ── Decision-context text patterns (from utils.js classifyUnknownPriority) ── */ + +const DECISION_PATTERNS = [ + /whether to/i, + /\bbuild\b/i, + /\blaunch\b/i, + /\bcontinue.*develop/i, + /\bproceed\b/i, +]; + +/* ── Evidence-related text patterns (from utils.js classifyUnknownPriority) ── */ + +const EVIDENCE_PATTERNS = [ + /evidence|metric|measure|criteria|validation|proof/i, +]; + +/* ── Normalise node label + description for text matching ── */ + +function normaliseText(value) { + return String(value || "") + .toLowerCase() + .replace(/[^a-z0-9]+/g, " ") + .trim(); +} + +function collectNodeText(node) { + return `${node?.label || ""} ${node?.description || ""}`.trim(); +} + +/* ── Collect connected node IDs (union of dependsOn, affects, childIds, and edges) ── */ + +function collectConnectedIds(node, graph) { + if (!node || !graph) return new Set(); + + const ids = new Set([ + ...(node.dependsOn || []), + ...(node.affects || []), + ...(node.childIds || []), + ]); + + if (node.parentId) ids.add(node.parentId); + + for (const edge of graph.edges || []) { + if (edge.fromNodeId === node.id) ids.add(edge.toNodeId); + if (edge.toNodeId === node.id) ids.add(edge.fromNodeId); + } + + return ids; +} + +/* ── Check whether any other unresolved unknown depends on this node ── */ + +function hasDownstreamUnknownDependents(node, graph, resolvedNodeIds) { + if (!node || !graph) return false; + + const resolvedSet = new Set(resolvedNodeIds || []); + const nodesById = new Map(graph.nodes.map((n) => [n.id, n])); + + // Check explicit dependsOn links pointing back to this node + for (const otherNode of graph.nodes) { + if (otherNode.id === node.id) continue; + if (otherNode.kind !== "unknown") continue; + if (resolvedSet.has(otherNode.id)) continue; + + if (otherNode.dependsOn.includes(node.id)) return true; + } + + // Check edge links where other unknown is source and this is target + for (const edge of graph.edges || []) { + if (edge.toNodeId !== node.id) continue; + const source = nodesById.get(edge.fromNodeId); + if (!source) continue; + if (source.kind !== "unknown") continue; + if (resolvedSet.has(source.id)) continue; + + return true; + } + + return false; +} + +/* ── Core classification function ── */ + +export function assessQuestionImportance(input) { + // Validate input contract + const { node, graph, resolvedNodeIds = [] } = input || {}; + if (!node || !graph || typeof node.kind !== "string") { + return { category: "cannot_determine", reason: "missing_input" }; + } + + const text = collectNodeText(node); + const connectedIds = collectConnectedIds(node, graph); + const isImportant = [ + hasDownstreamUnknownDependents(node, graph, resolvedNodeIds), + DECISION_PATTERNS.some((p) => p.test(text)), + ].some(Boolean); + + if (isImportant && connectedIds.size >= 1) { + return { category: "important" }; + } + + // Rule 2 — helpful + const isEvidenceText = EVIDENCE_PATTERNS.some((p) => p.test(text)); + if (isEvidenceText || connectedIds.size >= 2) { + return { category: "helpful" }; + } + + // Rule 4 — cannot_determine for empty nodes + if (!text.trim()) { + return { category: "cannot_determine", reason: "empty_node_text" }; + } + + // Rule 3 — default to incidental + return { category: "incidental" }; +} diff --git a/tests/graph/question-importance.test.js b/tests/graph/question-importance.test.js new file mode 100644 index 0000000..84b21c6 --- /dev/null +++ b/tests/graph/question-importance.test.js @@ -0,0 +1,279 @@ +/** + * Experiment 20 — Question Importance: passive classification test. + * + * Validates assessQuestionImportance against mock scenarios covering all four + * categories plus edge cases required by the experiment brief. + */ + +import { describe, expect, it } from "vitest"; +import { assessQuestionImportance } from "@/lib/graph/question-importance.js"; + +/* ── Test helpers ─────────────────────────────────────────────── */ + +function makeNode(id, kind = "unknown", status = "unknown") { + return { + id, label: `Test unknown ${id}`, description: `Description for ${id}`, kind, status, + confidence: "high", value: null, unit: null, evidenceIds: [], + dependsOn: [], affects: [], childIds: [], + }; +} + +function makeScenario(nodes, edges = [], resolved = [], active = null) { + return { + nodes, edges, resolvedNodeIds: resolved, activeUnknownNodeId: active, + centralStatement: "Test scenario", currentSummary: "Test", + reasoningState: null, + }; +} + +/* ── Category: important (downstream dependency) ─────────────── */ + +describe("assessQuestionImportance — important via downstream unknown", () => { + it("classifies a node as important when another unresolved unknown depends on it", () => { + const root = makeNode("root", "unknown", "unknown"); + const child = makeNode("child", "unknown", "unknown"); + // Add via edge so the classifier's edge traversal picks it up + const graph = makeScenario( + [root, child], + [{ id: "e-dep", fromNodeId: child.id, toNodeId: root.id, relationship: "depends_on", confidence: "medium", description: "" }], + ); + const result = assessQuestionImportance({ node: root, graph }); + + expect(result).toEqual({ category: "important" }); + }); + + it("does not classify as important when the dependent unknown is already resolved", () => { + const root = makeNode("root", "unknown", "unknown"); + const child = makeNode("child", "unknown", "resolved"); + child.dependsOn.push(root.id); + + const graph = makeScenario([root, child], [], ["child"]); + const result = assessQuestionImportance({ node: root, graph }); + + expect(result).not.toEqual({ category: "important" }); + }); +}); + +/* ── Category: important (decision text pattern) ─────────────── */ + +describe("assessQuestionImportance — important via decision text", () => { + it("classifies a node with 'whether to' as important when connected to the graph", () => { + const node = makeNode("d1", "unknown", "unknown"); + node.label = "Whether to build the feature"; + node.description = "Need to decide whether to build the feature for users."; + + const observer = makeNode("obs-1", "observation", "known"); + const graph = makeScenario([node, observer], [ + { id: "e-1", fromNodeId: "obs-1", toNodeId: node.id, relationship: "supports", confidence: "medium", description: "" }, + ]); + + const result = assessQuestionImportance({ node, graph }); + expect(result).toEqual({ category: "important" }); + }); + + it("classifies a node with 'build' as important when connected to the graph", () => { + const node = makeNode("d2", "unknown", "unknown"); + node.label = "Build approach for Confidence Engine"; + node.description = "Need to decide how to build the feature."; + + const observer = makeNode("obs-1", "observation", "known"); + const graph = makeScenario([node, observer], [ + { id: "e-1", fromNodeId: "obs-1", toNodeId: node.id, relationship: "supports", confidence: "medium", description: "" }, + ]); + + const result = assessQuestionImportance({ node, graph }); + expect(result).toEqual({ category: "important" }); + }); + + it("does not classify decision text as important when the node has zero connections", () => { + const node = makeNode("d3", "unknown", "unknown"); + node.label = "Whether to build the feature"; + node.description = ""; + + const graph = makeScenario([node]); + const result = assessQuestionImportance({ node, graph }); + expect(result.category).toBe("incidental"); + }); +}); + +/* ── Category: helpful (evidence text) ───────────────────────── */ + +describe("assessQuestionImportance — helpful via evidence text", () => { + it("classifies a node with 'evidence' as helpful", () => { + const node = makeNode("h1", "unknown", "unknown"); + node.label = "Evidence of adoption"; + node.description = "What evidence shows users adopt the feature."; + + const graph = makeScenario([node]); + const result = assessQuestionImportance({ node, graph }); + expect(result).toEqual({ category: "helpful" }); + }); + + it("classifies a node with 'metric' as helpful", () => { + const node = makeNode("h2", "unknown", "unknown"); + node.label = "Success metric"; + node.description = "Need to define the success criterion."; + + const graph = makeScenario([node]); + const result = assessQuestionImportance({ node, graph }); + expect(result).toEqual({ category: "helpful" }); + }); + + it("classifies a node with ≥2 connections as helpful even without evidence text", () => { + const node = makeNode("h3", "unknown", "unknown"); + node.label = "Supporting detail"; + node.description = "Contextual information for the decision."; + + const obs1 = makeNode("obs-1", "observation", "known"); + const obs2 = makeNode("obs-2", "observation", "known"); + + // This node is connected to both observers via edges + const graph = makeScenario( + [node, obs1, obs2], + [ + { id: "e-1", fromNodeId: "obs-1", toNodeId: node.id, relationship: "supports", confidence: "medium", description: "" }, + { id: "e-2", fromNodeId: "obs-2", toNodeId: node.id, relationship: "supports", confidence: "medium", description: "" }, + ], + ); + + const result = assessQuestionImportance({ node, graph }); + expect(result).toEqual({ category: "helpful" }); + }); +}); + +/* ── Category: incidental (default) ──────────────────────────── */ + +describe("assessQuestionImportance — incidental (default)", () => { + it("classifies a connected node as incidental when no patterns or dependencies match", () => { + const node = makeNode("inc-1", "unknown", "unknown"); + node.label = "Minor formatting detail"; + node.description = "How the display should look."; + + const obs = makeNode("obs-1", "observation", "known"); + const graph = makeScenario( + [node, obs], + [{ id: "e-1", fromNodeId: "obs-1", toNodeId: node.id, relationship: "supports", confidence: "medium", description: "" }], + ); + + const result = assessQuestionImportance({ node, graph }); + expect(result).toEqual({ category: "incidental" }); + }); + + it("classifies an unconnected unknown as incidental when text has no patterns", () => { + const node = makeNode("inc-2", "unknown", "unknown"); + node.label = "Color scheme preference"; + node.description = ""; + + const graph = makeScenario([node]); + const result = assessQuestionImportance({ node, graph }); + expect(result).toEqual({ category: "incidental" }); + }); +}); + +/* ── Edge case: cannot_determine (empty node text) ──────────── */ + +describe("assessQuestionImportance — cannot_determine", () => { + it("returns cannot_determine when both label and description are empty", () => { + const node = makeNode("empty-1", "unknown", "unknown"); + node.label = ""; + node.description = ""; + + const graph = makeScenario([node]); + const result = assessQuestionImportance({ node, graph }); + + expect(result.category).toBe("cannot_determine"); + }); +}); + +/* ── Edge case: missing input ────────────────────────────────── */ + +describe("assessQuestionImportance — missing input", () => { + it("returns cannot_determine when no input object is provided", () => { + const result = assessQuestionImportance(null); + expect(result.category).toBe("cannot_determine"); + }); + + it("returns cannot_determine when node.kind is not a string", () => { + const node = { id: "x", kind: null }; + const graph = makeScenario([node]); + const result = assessQuestionImportance({ node, graph }); + expect(result.category).toBe("cannot_determine"); + }); + + it("returns cannot_determine when graph is missing", () => { + const node = makeNode("x", "unknown", "unknown"); + const result = assessQuestionImportance({ node }); + expect(result.category).toBe("cannot_determine"); + }); +}); + +/* ── Determinism ─────────────────────────────────────────────── */ + +describe("assessQuestionImportance — deterministic output", () => { + it("produces the same result across multiple identical calls", () => { + const node = makeNode("det-1", "unknown", "unknown"); + node.label = "Evidence of adoption"; + node.description = ""; + + const graph = makeScenario([node]); + + const r1 = assessQuestionImportance({ node, graph }); + const r2 = assessQuestionImportance({ node, graph }); + + expect(r1).toEqual(r2); + }); +}); + +/* ── Input immutability ──────────────────────────────────────── */ + +describe("assessQuestionImportance — input not mutated", () => { + it("does not mutate the input node", () => { + const node = makeNode("imm-1", "unknown", "unknown"); + node.label = "Evidence of adoption"; + node.description = ""; + const originalLabel = node.label; + const originalDependsOn = [...node.dependsOn]; + + const graph = makeScenario([node]); + assessQuestionImportance({ node, graph }); + + expect(node.label).toBe(originalLabel); + expect(node.dependsOn).toEqual(originalDependsOn); + }); + + it("does not mutate the input graph", () => { + const node = makeNode("imm-2", "unknown", "unknown"); + node.label = "Evidence of adoption"; + node.description = ""; + + const obs = makeNode("obs-1", "observation", "known"); + const graph = makeScenario([node, obs]); + + const originalNodes = JSON.parse(JSON.stringify(graph.nodes)); + const originalEdges = JSON.parse(JSON.stringify(graph.edges)); + + assessQuestionImportance({ node, graph }); + + expect(JSON.stringify(graph.nodes)).toBe(JSON.stringify(originalNodes)); + expect(JSON.stringify(graph.edges)).toBe(JSON.stringify(originalEdges)); + }); +}); + +/* ── Rule interaction: downstream depends_on edge-based ──────── */ + +describe("assessQuestionImportance — via graph edges", () => { + it("detects downstream dependency through an edge relationship (unknown → unknown)", () => { + const parent = makeNode("parent-unk", "unknown", "unknown"); + const child = makeNode("child-unk", "unknown", "unknown"); + + const graph = makeScenario( + [parent, child], + [ + { id: "e-dep", fromNodeId: child.id, toNodeId: parent.id, relationship: "depends_on", confidence: "medium", description: "" }, + ], + ); + + const result = assessQuestionImportance({ node: parent, graph }); + expect(result).toEqual({ category: "important" }); + }); +});