experiment: evaluate question importance across long investigation

tests/graph/question-importance.long-investigation.test.js

@@ -0,0 +1,160 @@
+/**
+ * Experiment 20 — Question Importance: long-investigation evaluation test.
+ *
+ * Runs the passive classifier across every unresolved unknown at each turn of
+ * the long-investigation fixture (longTurns). Captures per-turn classifications
+ * and produces a category distribution summary. Validates correctness requirements
+ * from the experiment brief.
+ */
+
+import { describe, expect, it } from "vitest";
+import { assessQuestionImportance } from "@/lib/graph/question-importance.js";
+import { buildScenarioFixture } from "@/lib/mocks/scenarios.js";
+
+/* ── The long-investigation fixture (5 turns) ─────────── */
+
+describe("assessQuestionImportance — long investigation evaluation", () => {
+  const scenarioName = "long";
+  const turnCount = 5;
+
+  /* Run the full sequence and collect results */
+  const allResults = [];
+
+  for (let t = 0; t < turnCount; t++) {
+    const fixture = buildScenarioFixture(scenarioName, t);
+    expect(fixture).not.toBeNull(`Turn ${t} should have a valid fixture`);
+
+    const graph = fixture.situationGraph;
+    const resolvedIds = new Set(graph.resolvedNodeIds || []);
+
+    /* Evaluate every unresolved unknown at this turn */
+    for (const node of graph.nodes) {
+      if (node.kind !== "unknown") continue;
+      if (resolvedIds.has(node.id)) continue;
+
+      const result = assessQuestionImportance({ node, graph, resolvedNodeIds: graph.resolvedNodeIds });
+
+      allResults.push({ turn: t, nodeId: node.id, label: node.label, category: result.category });
+    }
+  }
+
+  /* ── Assertion: the sequence runs without error and every unknown gets a valid classification */
+
+  it("every unresolved unknown across all turns receives a valid classification", () => {
+    const validCategories = ["important", "helpful", "incidental", "cannot_determine"];
+    for (const r of allResults) {
+      expect(validCategories).toContain(r.category);
+    }
+  });
+
+  it("the complete long-investigation sequence runs without error and produces results", () => {
+    expect(allResults.length).toBeGreaterThan(0);
+  });
+
+  /* ── Category distribution summary ───────────────────── */
+
+  const categories = {};
+  for (const r of allResults) {
+    categories[r.category] = (categories[r.category] || 0) + 1;
+  }
+
+  it("category distribution across the full sequence", () => {
+    console.log("\n=== Long-Investigation Classification Results ===");
+    console.log(`Total unresolved unknowns classified: ${allResults.length}`);
+    console.log("Category distribution:", JSON.stringify(categories, null, 2));
+    expect(allResults.length).toBeGreaterThan(0);
+  });
+
+  /* ── Per-turn detail (for evaluation review) ─────────── */
+
+  it("detailed per-turn classifications", () => {
+    const byTurn = {};
+    for (const r of allResults) {
+      if (!byTurn[r.turn]) byTurn[r.turn] = [];
+      byTurn[r.turn].push(r);
+    }
+    for (const [turn, results] of Object.entries(byTurn)) {
+      console.log(`\n--- Turn ${turn} (${results.length} unresolved unknowns) ---`);
+      for (const r of results) {
+        const shortLabel = r.label?.length > 60 ? r.label.slice(0, 60) + "..." : r.label;
+        console.log(`  [${r.category}] ${shortLabel}`);
+      }
+    }
+    expect(allResults.length).toBeGreaterThan(0);
+  });
+
+  /* ── Requirement: at least two distinct categories or explicit collapse ───── */
+
+  it("produces more than one meaningful category (or explicitly records collapse)", () => {
+    const distinct = new Set(allResults.map((r) => r.category));
+    if (distinct.size < 2) {
+      console.log("\nNote: classification collapsed to a single category:", [...distinct][0]);
+    }
+    expect(distinct.size).toBeGreaterThanOrEqual(1);
+  });
+
+  /* ── Determinism ─────────────────────────────────────── */
+
+  it("identical turn data produces identical results", () => {
+    const fixture = buildScenarioFixture(scenarioName, 2);
+    const graph = fixture.situationGraph;
+
+    for (const node of graph.nodes) {
+      if (node.kind !== "unknown") continue;
+      if ((graph.resolvedNodeIds || []).includes(node.id)) continue;
+
+      const r1 = assessQuestionImportance({ node, graph });
+      const deepCopy = JSON.parse(JSON.stringify(graph));
+      const r2 = assessQuestionImportance({ node: { ...node }, graph: deepCopy });
+      expect(r1).toEqual(r2);
+    }
+  });
+
+  /* ── Input immutability ─────────────────────────────── */
+
+  it("does not mutate input nodes or graphs during the sequence", () => {
+    const fixture = buildScenarioFixture(scenarioName, 3);
+    const originalNodesJSON = JSON.stringify(fixture.situationGraph.nodes);
+    const originalEdgesJSON = JSON.stringify(fixture.situationGraph.edges);
+
+    for (const node of fixture.situationGraph.nodes) {
+      if (node.kind !== "unknown") continue;
+      if ((fixture.situationGraph.resolvedNodeIds || []).includes(node.id)) continue;
+      const nodeSnapshot = JSON.parse(JSON.stringify(node));
+      assessQuestionImportance({ node, graph: fixture.situationGraph });
+
+      /* original node must be unchanged */
+      expect(node).toEqual(nodeSnapshot);
+    }
+
+    expect(JSON.stringify(fixture.situationGraph.nodes)).toBe(originalNodesJSON);
+    expect(JSON.stringify(fixture.situationGraph.edges)).toBe(originalEdgesJSON);
+  });
+
+  /* ── Evaluation: key questions for manual review ─────── */
+
+  it("key classification examples for evaluation", () => {
+    const important = [];
+    const helpful = [];
+    const incidental = [];
+    const cannot_determine = [];
+
+    for (const r of allResults) {
+      const entry = { turn: r.turn, id: r.nodeId, label: r.label, category: r.category };
+      if (r.category === "important") important.push(entry);
+      else if (r.category === "helpful") helpful.push(entry);
+      else if (r.category === "incidental") incidental.push(entry);
+      else cannot_determine.push(entry);
+    }
+
+    console.log("\n=== Detailed Evaluation Data ===");
+    console.log("IMPORTANT:", JSON.stringify(important, null, 2));
+    console.log("HELPFUL:", JSON.stringify(helpful, null, 2));
+    console.log("INCIDENTAL:", JSON.stringify(incidental, null, 2));
+    console.log("CANNOT_DETERMINE:", JSON.stringify(cannot_determine, null, 2));
+
+    expect(important.length).toBeGreaterThanOrEqual(0);
+    expect(helpful.length).toBeGreaterThanOrEqual(0);
+    expect(incidental.length).toBeGreaterThanOrEqual(0);
+  });
+});