experiment: add passive question importance test
This commit is contained in:
@@ -0,0 +1,279 @@
|
||||
/**
|
||||
* Experiment 20 — Question Importance: passive classification test.
|
||||
*
|
||||
* Validates assessQuestionImportance against mock scenarios covering all four
|
||||
* categories plus edge cases required by the experiment brief.
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { assessQuestionImportance } from "@/lib/graph/question-importance.js";
|
||||
|
||||
/* ── Test helpers ─────────────────────────────────────────────── */
|
||||
|
||||
function makeNode(id, kind = "unknown", status = "unknown") {
|
||||
return {
|
||||
id, label: `Test unknown ${id}`, description: `Description for ${id}`, kind, status,
|
||||
confidence: "high", value: null, unit: null, evidenceIds: [],
|
||||
dependsOn: [], affects: [], childIds: [],
|
||||
};
|
||||
}
|
||||
|
||||
function makeScenario(nodes, edges = [], resolved = [], active = null) {
|
||||
return {
|
||||
nodes, edges, resolvedNodeIds: resolved, activeUnknownNodeId: active,
|
||||
centralStatement: "Test scenario", currentSummary: "Test",
|
||||
reasoningState: null,
|
||||
};
|
||||
}
|
||||
|
||||
/* ── Category: important (downstream dependency) ─────────────── */
|
||||
|
||||
describe("assessQuestionImportance — important via downstream unknown", () => {
|
||||
it("classifies a node as important when another unresolved unknown depends on it", () => {
|
||||
const root = makeNode("root", "unknown", "unknown");
|
||||
const child = makeNode("child", "unknown", "unknown");
|
||||
// Add via edge so the classifier's edge traversal picks it up
|
||||
const graph = makeScenario(
|
||||
[root, child],
|
||||
[{ id: "e-dep", fromNodeId: child.id, toNodeId: root.id, relationship: "depends_on", confidence: "medium", description: "" }],
|
||||
);
|
||||
const result = assessQuestionImportance({ node: root, graph });
|
||||
|
||||
expect(result).toEqual({ category: "important" });
|
||||
});
|
||||
|
||||
it("does not classify as important when the dependent unknown is already resolved", () => {
|
||||
const root = makeNode("root", "unknown", "unknown");
|
||||
const child = makeNode("child", "unknown", "resolved");
|
||||
child.dependsOn.push(root.id);
|
||||
|
||||
const graph = makeScenario([root, child], [], ["child"]);
|
||||
const result = assessQuestionImportance({ node: root, graph });
|
||||
|
||||
expect(result).not.toEqual({ category: "important" });
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Category: important (decision text pattern) ─────────────── */
|
||||
|
||||
describe("assessQuestionImportance — important via decision text", () => {
|
||||
it("classifies a node with 'whether to' as important when connected to the graph", () => {
|
||||
const node = makeNode("d1", "unknown", "unknown");
|
||||
node.label = "Whether to build the feature";
|
||||
node.description = "Need to decide whether to build the feature for users.";
|
||||
|
||||
const observer = makeNode("obs-1", "observation", "known");
|
||||
const graph = makeScenario([node, observer], [
|
||||
{ id: "e-1", fromNodeId: "obs-1", toNodeId: node.id, relationship: "supports", confidence: "medium", description: "" },
|
||||
]);
|
||||
|
||||
const result = assessQuestionImportance({ node, graph });
|
||||
expect(result).toEqual({ category: "important" });
|
||||
});
|
||||
|
||||
it("classifies a node with 'build' as important when connected to the graph", () => {
|
||||
const node = makeNode("d2", "unknown", "unknown");
|
||||
node.label = "Build approach for Confidence Engine";
|
||||
node.description = "Need to decide how to build the feature.";
|
||||
|
||||
const observer = makeNode("obs-1", "observation", "known");
|
||||
const graph = makeScenario([node, observer], [
|
||||
{ id: "e-1", fromNodeId: "obs-1", toNodeId: node.id, relationship: "supports", confidence: "medium", description: "" },
|
||||
]);
|
||||
|
||||
const result = assessQuestionImportance({ node, graph });
|
||||
expect(result).toEqual({ category: "important" });
|
||||
});
|
||||
|
||||
it("does not classify decision text as important when the node has zero connections", () => {
|
||||
const node = makeNode("d3", "unknown", "unknown");
|
||||
node.label = "Whether to build the feature";
|
||||
node.description = "";
|
||||
|
||||
const graph = makeScenario([node]);
|
||||
const result = assessQuestionImportance({ node, graph });
|
||||
expect(result.category).toBe("incidental");
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Category: helpful (evidence text) ───────────────────────── */
|
||||
|
||||
describe("assessQuestionImportance — helpful via evidence text", () => {
|
||||
it("classifies a node with 'evidence' as helpful", () => {
|
||||
const node = makeNode("h1", "unknown", "unknown");
|
||||
node.label = "Evidence of adoption";
|
||||
node.description = "What evidence shows users adopt the feature.";
|
||||
|
||||
const graph = makeScenario([node]);
|
||||
const result = assessQuestionImportance({ node, graph });
|
||||
expect(result).toEqual({ category: "helpful" });
|
||||
});
|
||||
|
||||
it("classifies a node with 'metric' as helpful", () => {
|
||||
const node = makeNode("h2", "unknown", "unknown");
|
||||
node.label = "Success metric";
|
||||
node.description = "Need to define the success criterion.";
|
||||
|
||||
const graph = makeScenario([node]);
|
||||
const result = assessQuestionImportance({ node, graph });
|
||||
expect(result).toEqual({ category: "helpful" });
|
||||
});
|
||||
|
||||
it("classifies a node with ≥2 connections as helpful even without evidence text", () => {
|
||||
const node = makeNode("h3", "unknown", "unknown");
|
||||
node.label = "Supporting detail";
|
||||
node.description = "Contextual information for the decision.";
|
||||
|
||||
const obs1 = makeNode("obs-1", "observation", "known");
|
||||
const obs2 = makeNode("obs-2", "observation", "known");
|
||||
|
||||
// This node is connected to both observers via edges
|
||||
const graph = makeScenario(
|
||||
[node, obs1, obs2],
|
||||
[
|
||||
{ id: "e-1", fromNodeId: "obs-1", toNodeId: node.id, relationship: "supports", confidence: "medium", description: "" },
|
||||
{ id: "e-2", fromNodeId: "obs-2", toNodeId: node.id, relationship: "supports", confidence: "medium", description: "" },
|
||||
],
|
||||
);
|
||||
|
||||
const result = assessQuestionImportance({ node, graph });
|
||||
expect(result).toEqual({ category: "helpful" });
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Category: incidental (default) ──────────────────────────── */
|
||||
|
||||
describe("assessQuestionImportance — incidental (default)", () => {
|
||||
it("classifies a connected node as incidental when no patterns or dependencies match", () => {
|
||||
const node = makeNode("inc-1", "unknown", "unknown");
|
||||
node.label = "Minor formatting detail";
|
||||
node.description = "How the display should look.";
|
||||
|
||||
const obs = makeNode("obs-1", "observation", "known");
|
||||
const graph = makeScenario(
|
||||
[node, obs],
|
||||
[{ id: "e-1", fromNodeId: "obs-1", toNodeId: node.id, relationship: "supports", confidence: "medium", description: "" }],
|
||||
);
|
||||
|
||||
const result = assessQuestionImportance({ node, graph });
|
||||
expect(result).toEqual({ category: "incidental" });
|
||||
});
|
||||
|
||||
it("classifies an unconnected unknown as incidental when text has no patterns", () => {
|
||||
const node = makeNode("inc-2", "unknown", "unknown");
|
||||
node.label = "Color scheme preference";
|
||||
node.description = "";
|
||||
|
||||
const graph = makeScenario([node]);
|
||||
const result = assessQuestionImportance({ node, graph });
|
||||
expect(result).toEqual({ category: "incidental" });
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Edge case: cannot_determine (empty node text) ──────────── */
|
||||
|
||||
describe("assessQuestionImportance — cannot_determine", () => {
|
||||
it("returns cannot_determine when both label and description are empty", () => {
|
||||
const node = makeNode("empty-1", "unknown", "unknown");
|
||||
node.label = "";
|
||||
node.description = "";
|
||||
|
||||
const graph = makeScenario([node]);
|
||||
const result = assessQuestionImportance({ node, graph });
|
||||
|
||||
expect(result.category).toBe("cannot_determine");
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Edge case: missing input ────────────────────────────────── */
|
||||
|
||||
describe("assessQuestionImportance — missing input", () => {
|
||||
it("returns cannot_determine when no input object is provided", () => {
|
||||
const result = assessQuestionImportance(null);
|
||||
expect(result.category).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("returns cannot_determine when node.kind is not a string", () => {
|
||||
const node = { id: "x", kind: null };
|
||||
const graph = makeScenario([node]);
|
||||
const result = assessQuestionImportance({ node, graph });
|
||||
expect(result.category).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("returns cannot_determine when graph is missing", () => {
|
||||
const node = makeNode("x", "unknown", "unknown");
|
||||
const result = assessQuestionImportance({ node });
|
||||
expect(result.category).toBe("cannot_determine");
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Determinism ─────────────────────────────────────────────── */
|
||||
|
||||
describe("assessQuestionImportance — deterministic output", () => {
|
||||
it("produces the same result across multiple identical calls", () => {
|
||||
const node = makeNode("det-1", "unknown", "unknown");
|
||||
node.label = "Evidence of adoption";
|
||||
node.description = "";
|
||||
|
||||
const graph = makeScenario([node]);
|
||||
|
||||
const r1 = assessQuestionImportance({ node, graph });
|
||||
const r2 = assessQuestionImportance({ node, graph });
|
||||
|
||||
expect(r1).toEqual(r2);
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Input immutability ──────────────────────────────────────── */
|
||||
|
||||
describe("assessQuestionImportance — input not mutated", () => {
|
||||
it("does not mutate the input node", () => {
|
||||
const node = makeNode("imm-1", "unknown", "unknown");
|
||||
node.label = "Evidence of adoption";
|
||||
node.description = "";
|
||||
const originalLabel = node.label;
|
||||
const originalDependsOn = [...node.dependsOn];
|
||||
|
||||
const graph = makeScenario([node]);
|
||||
assessQuestionImportance({ node, graph });
|
||||
|
||||
expect(node.label).toBe(originalLabel);
|
||||
expect(node.dependsOn).toEqual(originalDependsOn);
|
||||
});
|
||||
|
||||
it("does not mutate the input graph", () => {
|
||||
const node = makeNode("imm-2", "unknown", "unknown");
|
||||
node.label = "Evidence of adoption";
|
||||
node.description = "";
|
||||
|
||||
const obs = makeNode("obs-1", "observation", "known");
|
||||
const graph = makeScenario([node, obs]);
|
||||
|
||||
const originalNodes = JSON.parse(JSON.stringify(graph.nodes));
|
||||
const originalEdges = JSON.parse(JSON.stringify(graph.edges));
|
||||
|
||||
assessQuestionImportance({ node, graph });
|
||||
|
||||
expect(JSON.stringify(graph.nodes)).toBe(JSON.stringify(originalNodes));
|
||||
expect(JSON.stringify(graph.edges)).toBe(JSON.stringify(originalEdges));
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Rule interaction: downstream depends_on edge-based ──────── */
|
||||
|
||||
describe("assessQuestionImportance — via graph edges", () => {
|
||||
it("detects downstream dependency through an edge relationship (unknown → unknown)", () => {
|
||||
const parent = makeNode("parent-unk", "unknown", "unknown");
|
||||
const child = makeNode("child-unk", "unknown", "unknown");
|
||||
|
||||
const graph = makeScenario(
|
||||
[parent, child],
|
||||
[
|
||||
{ id: "e-dep", fromNodeId: child.id, toNodeId: parent.id, relationship: "depends_on", confidence: "medium", description: "" },
|
||||
],
|
||||
);
|
||||
|
||||
const result = assessQuestionImportance({ node: parent, graph });
|
||||
expect(result).toEqual({ category: "important" });
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user