experiment: test question relevance against decision target
This commit is contained in:
@@ -0,0 +1,308 @@
|
||||
/**
|
||||
* Experiment 21 — Question Relevance to Decision Target.
|
||||
*
|
||||
* Tests assessQuestionRelevanceToDecision against explicit decision targets
|
||||
* and the long-investigation scenario fixture.
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { assessQuestionRelevanceToDecision } from "@/lib/graph/question-decision-relevance.js";
|
||||
import { buildScenarioFixture } from "@/lib/mocks/scenarios.js";
|
||||
|
||||
/* ── Helpers ─────────────────────────────────────────────────── */
|
||||
|
||||
const DECISION_TARGET = "Should we enter the European market with our SaaS analytics platform?";
|
||||
|
||||
function makeUnknown(id, label) {
|
||||
return {
|
||||
id, label: label || `Unknown ${id}`, description: label || `Unknown ${id}`,
|
||||
kind: "unknown", status: "unknown", confidence: "low",
|
||||
value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], childIds: [],
|
||||
};
|
||||
}
|
||||
|
||||
function makeGraphFor(nodes) {
|
||||
return {
|
||||
nodes, edges: [], resolvedNodeIds: [], activeUnknownNodeId: null,
|
||||
centralStatement: "Test", currentSummary: "Test", reasoningState: null,
|
||||
};
|
||||
}
|
||||
|
||||
/* ── Could change decision ───────────────────────────────────── */
|
||||
|
||||
describe("assessQuestionRelevanceToDecision — could_change_decision", () => {
|
||||
it("classifies a direct go/no-go question against the decision target", () => {
|
||||
const result = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
unknown: makeUnknown("g1", "Whether to proceed with European market entry"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("could_change_decision");
|
||||
expect(typeof result.reason).toBe("string");
|
||||
expect(result.reason.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("classifies a 'whether we should enter' question as could_change_decision", () => {
|
||||
const result = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
unknown: makeUnknown("g2", "Whether we should enter the European market"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("could_change_decision");
|
||||
});
|
||||
|
||||
it("is deterministic for identical inputs", () => {
|
||||
const node = makeUnknown("g3", "Whether to proceed with expansion into Europe");
|
||||
const input = { decisionTarget: DECISION_TARGET, unknown: node };
|
||||
const r1 = assessQuestionRelevanceToDecision(input);
|
||||
const r2 = assessQuestionRelevanceToDecision(input);
|
||||
expect(r1).toEqual(r2);
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Supports decision ───────────────────────────────────────── */
|
||||
|
||||
describe("assessQuestionRelevanceToDecision — supports_decision", () => {
|
||||
it("classifies a compliance question as supports_decision (necessary precondition)", () => {
|
||||
const result = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
unknown: makeUnknown("c1", "Whether our product is suitable for European compliance requirements"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("supports_decision");
|
||||
});
|
||||
|
||||
it("classifies a cost justification question as supports_decision (feasibility)", () => {
|
||||
const result = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
unknown: makeUnknown("c2", "Whether the cost of achieving compliance is justified by the market size"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("supports_decision");
|
||||
});
|
||||
|
||||
it("classifies a differentiation question as supports_decision (supporting context)", () => {
|
||||
const result = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
unknown: makeUnknown("c3", "Whether we have competitive differentiation against existing European players"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("supports_decision");
|
||||
});
|
||||
|
||||
it("classifies a feasibility question as supports_decision even without precondition keywords", () => {
|
||||
const result = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
unknown: makeUnknown("c4", "Is the investment of $500K and six months justified by potential returns"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("supports_decision");
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Unlikely to change decision ─────────────────────────────── */
|
||||
|
||||
describe("assessQuestionRelevanceToDecision — unlikely_to_change_decision", () => {
|
||||
it("classifies a background comparison as unlikely_to_change_decision", () => {
|
||||
const result = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
unknown: makeUnknown("b1", "What benchmarks do other SaaS companies use for market sizing"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("unlikely_to_change_decision");
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Cannot determine — missing input ────────────────────────── */
|
||||
|
||||
describe("assessQuestionRelevanceToDecision — cannot_determine (missing input)", () => {
|
||||
it("returns cannot_determine when decisionTarget is missing", () => {
|
||||
const result = assessQuestionRelevanceToDecision({
|
||||
unknown: makeUnknown("m1", "Whether to enter Europe"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("returns cannot_determine when decisionTarget is empty string", () => {
|
||||
const result = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: "",
|
||||
unknown: makeUnknown("m2", "Whether to enter Europe"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("returns cannot_determine when unknown is missing", () => {
|
||||
const result = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("returns cannot_determine when unknown has empty label and description", () => {
|
||||
const node = makeUnknown("m3", "");
|
||||
node.label = "";
|
||||
node.description = "";
|
||||
|
||||
const result = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
unknown: node,
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("returns cannot_determine when no input object is provided", () => {
|
||||
const result = assessQuestionRelevanceToDecision(null);
|
||||
expect(result.relevance).toBe("cannot_determine");
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Input immutability ──────────────────────────────────────── */
|
||||
|
||||
describe("assessQuestionRelevanceToDecision — input immutability", () => {
|
||||
it("does not mutate the unknown node", () => {
|
||||
const node = makeUnknown("i1", "Whether to proceed with European market entry");
|
||||
const snapshot = JSON.parse(JSON.stringify(node));
|
||||
assessQuestionRelevanceToDecision({ decisionTarget: DECISION_TARGET, unknown: node });
|
||||
expect(node).toEqual(snapshot);
|
||||
});
|
||||
|
||||
it("does not mutate the graph (when provided)", () => {
|
||||
const node = makeUnknown("i2", "Whether to proceed with European market entry");
|
||||
const graph = makeGraphFor([node]);
|
||||
const originalNodesJSON = JSON.stringify(graph.nodes);
|
||||
assessQuestionRelevanceToDecision({ decisionTarget: DECISION_TARGET, unknown: node, graph });
|
||||
expect(JSON.stringify(graph.nodes)).toBe(originalNodesJSON);
|
||||
});
|
||||
|
||||
it("does not mutate the decisionTarget string", () => {
|
||||
const dt = "Should we enter the European market with our SaaS analytics platform?";
|
||||
assessQuestionRelevanceToDecision({ decisionTarget: dt, unknown: makeUnknown("i3", "test") });
|
||||
expect(dt).toBe("Should we enter the European market with our SaaS analytics platform?");
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Complete long-investigation sequence ─────────────────────── */
|
||||
|
||||
describe("assessQuestionRelevanceToDecision — long investigation sequence", () => {
|
||||
const scenarioName = "long";
|
||||
const turnCount = 5;
|
||||
|
||||
const allResults = [];
|
||||
|
||||
for (let t = 0; t < turnCount; t++) {
|
||||
const fixture = buildScenarioFixture(scenarioName, t);
|
||||
expect(fixture).not.toBeNull(`Turn ${t} should have a valid fixture`);
|
||||
|
||||
// Use the centralStatement as decision target (present in every turn)
|
||||
const decisionTarget = fixture.situationGraph.centralStatement;
|
||||
const resolvedIds = new Set(fixture.situationGraph.resolvedNodeIds || []);
|
||||
|
||||
for (const node of fixture.situationGraph.nodes) {
|
||||
if (node.kind !== "unknown") continue;
|
||||
if (resolvedIds.has(node.id)) continue;
|
||||
|
||||
const result = assessQuestionRelevanceToDecision({
|
||||
decisionTarget,
|
||||
unknown: node,
|
||||
graph: fixture.situationGraph,
|
||||
});
|
||||
|
||||
allResults.push({ turn: t, nodeId: node.id, label: node.label, relevance: result.relevance, reason: result.reason });
|
||||
}
|
||||
}
|
||||
|
||||
it("every unresolved unknown across all turns receives a valid relevance classification", () => {
|
||||
const validCategories = ["could_change_decision", "supports_decision", "unlikely_to_change_decision", "cannot_determine"];
|
||||
for (const r of allResults) {
|
||||
expect(validCategories).toContain(r.relevance);
|
||||
}
|
||||
});
|
||||
|
||||
it("the complete long-investigation sequence runs without error and produces results", () => {
|
||||
expect(allResults.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("produces more than one distinct relevance category across the sequence", () => {
|
||||
const distinct = new Set(allResults.map((r) => r.relevance));
|
||||
expect(distinct.size).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("category distribution across the full sequence", () => {
|
||||
const categories = {};
|
||||
for (const r of allResults) {
|
||||
categories[r.relevance] = (categories[r.relevance] || 0) + 1;
|
||||
}
|
||||
console.log("\n=== Experiment 21 — Classification Results ===");
|
||||
console.log(`Total unresolved unknowns classified: ${allResults.length}`);
|
||||
console.log("Category distribution:", JSON.stringify(categories, null, 2));
|
||||
expect(allResults.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("detailed per-turn classifications", () => {
|
||||
const byTurn = {};
|
||||
for (const r of allResults) {
|
||||
if (!byTurn[r.turn]) byTurn[r.turn] = [];
|
||||
byTurn[r.turn].push(r);
|
||||
}
|
||||
for (const [turn, results] of Object.entries(byTurn)) {
|
||||
console.log(`\n--- Turn ${turn} (${results.length} unresolved unknowns) ---`);
|
||||
for (const r of results) {
|
||||
const shortLabel = r.label?.length > 60 ? r.label.slice(0, 60) + "..." : r.label;
|
||||
console.log(` [${r.relevance}] ${shortLabel}`);
|
||||
}
|
||||
}
|
||||
expect(allResults.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("Turn 0 (market demand question) is classified as could_change_decision", () => {
|
||||
const turn0 = allResults.find((r) => r.turn === 0);
|
||||
expect(turn0).toBeDefined();
|
||||
expect(turn0.relevance).toBe("could_change_decision");
|
||||
});
|
||||
|
||||
it("Turns 1-3 (compliance, cost, differentiation) are classified as supports_decision", () => {
|
||||
const turns1to3 = allResults.filter((r) => r.turn >= 1 && r.turn <= 3);
|
||||
for (const r of turns1to3) {
|
||||
expect(r.relevance).toBe("supports_decision");
|
||||
}
|
||||
});
|
||||
|
||||
it("all classified questions include a non-empty reason", () => {
|
||||
for (const r of allResults) {
|
||||
if (r.relevance !== "cannot_determine") {
|
||||
expect(typeof r.reason).toBe("string");
|
||||
expect(r.reason.length).toBeGreaterThan(0);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Comparison with Experiment 20: different categories produced ─ */
|
||||
|
||||
describe("assessQuestionRelevanceToDecision — comparison with Experiment 20 collapse", () => {
|
||||
it("does not collapse entirely to one category across the long investigation", () => {
|
||||
const scenarioName = "long";
|
||||
const results = [];
|
||||
|
||||
for (let t = 0; t < 5; t++) {
|
||||
const fixture = buildScenarioFixture(scenarioName, t);
|
||||
if (!fixture) continue;
|
||||
const decisionTarget = fixture.situationGraph.centralStatement;
|
||||
const resolvedIds = new Set(fixture.situationGraph.resolvedNodeIds || []);
|
||||
|
||||
for (const node of fixture.situationGraph.nodes) {
|
||||
if (node.kind !== "unknown") continue;
|
||||
if (resolvedIds.has(node.id)) continue;
|
||||
|
||||
results.push(assessQuestionRelevanceToDecision({ decisionTarget, unknown: node }));
|
||||
}
|
||||
}
|
||||
|
||||
const distinct = new Set(results.map((r) => r.relevance));
|
||||
expect(distinct.size).toBeGreaterThan(1);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user