experiment: test questions against decision conditions
This commit is contained in:
@@ -1198,6 +1198,23 @@ Run the classifier passively across the same long-investigation turns used in Ex
|
||||
|
||||
---
|
||||
|
||||
### Experiment 22 — Question Relevance Against Explicit Decision Conditions
|
||||
|
||||
Explicit decision conditions were supplied:
|
||||
|
||||
1. Credible customer demand exists in Europe
|
||||
2. European compliance is achievable
|
||||
3. The expected market value justifies the cost of entry
|
||||
4. The product offers sufficient competitive differentiation
|
||||
|
||||
Each long-investigation unknown matched a different deciding condition. All four correctly classified as `tests_deciding_condition`.
|
||||
|
||||
Category variety is not automatically a measure of quality — here, uniformity (all four as decisive) is correct because each question directly tests a required condition.
|
||||
|
||||
The classifier remains passive and is not in the active reasoning path.
|
||||
|
||||
---
|
||||
|
||||
## Current Open Questions
|
||||
|
||||
The following are active explorations rather than decisions.
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
/**
|
||||
* Experiment 22 — Question Relevance Against Decision Conditions.
|
||||
*
|
||||
* Classifies an unresolved unknown against explicit decision conditions
|
||||
* that define what must be true for a specific decision to be sensible.
|
||||
*
|
||||
* Classification categories:
|
||||
* tests_deciding_condition — The question directly tests something
|
||||
* required for the decision's justification.
|
||||
* adds_supporting_evidence — The answer would strengthen confidence
|
||||
* but doesn't test a required condition.
|
||||
* outside_decision_conditions — Not meaningfully connected to any condition.
|
||||
* cannot_determine — Inputs are missing, empty, or too unclear.
|
||||
*
|
||||
* No LLM calls. No new graph fields. Pure function. No engine mutation.
|
||||
*/
|
||||
|
||||
function normalise(value) {
|
||||
return String(value || "").toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
|
||||
}
|
||||
|
||||
/* ── Primary concept groups (substring-based detection) ─*/
|
||||
|
||||
const DEMAND_CONCEPTS = ["demand","need","interest","customers","audience"];
|
||||
const COMPLIANCE_CONCEPTS = ["compliance","regulation","legal","required","mandatory","gdpr","data residency"];
|
||||
const VALUE_COST_CONCEPTS = ["cost","investment","justif","return","viability","financial"];
|
||||
const DIFFERENTIATION_CONCEPTS = ["differentiat","advantage","competit","positioning","superior","unique"];
|
||||
|
||||
/* ── Supporting evidence concept groups (broader, indirect terms) ─*/
|
||||
|
||||
const DEMAND_SUPPORT_CONCEPTS = ["geography","region","country","territory","area","locale","segment","target","entry","expansion","penetration"];
|
||||
const COMPLIANCE_SUPPORT_CONCEPTS = ["privacy","certification","standards"];
|
||||
const VALUE_COST_SUPPORT_CONCEPTS = ["budget","price","revenue","pricing","resource","structure","subsidiary"];
|
||||
const DIFFERENTIATION_SUPPORT_CONCEPTS = ["edge","distinct","feature","benefit"];
|
||||
|
||||
const CATEGORY_GROUPS = {
|
||||
demand: DEMAND_CONCEPTS,
|
||||
compliance: COMPLIANCE_CONCEPTS,
|
||||
value_cost: VALUE_COST_CONCEPTS,
|
||||
differentiation: DIFFERENTIATION_CONCEPTS,
|
||||
};
|
||||
|
||||
const SUPPORT_MAP = {
|
||||
demand: DEMAND_SUPPORT_CONCEPTS,
|
||||
compliance: COMPLIANCE_SUPPORT_CONCEPTS,
|
||||
value_cost: VALUE_COST_SUPPORT_CONCEPTS,
|
||||
differentiation: DIFFERENTIATION_SUPPORT_CONCEPTS,
|
||||
};
|
||||
|
||||
/* ── Which primary concept categories does a condition mention? ─*/
|
||||
|
||||
function getConditionCategories(condition) {
|
||||
const lower = condition.toLowerCase();
|
||||
const cats = [];
|
||||
for (const [name, concepts] of Object.entries(CATEGORY_GROUPS)) {
|
||||
if (concepts.some((kw) => lower.includes(normalise(kw)))) cats.push(name);
|
||||
}
|
||||
return cats;
|
||||
}
|
||||
|
||||
/* ── Primary categories a question draws from (substring matching) ─*/
|
||||
|
||||
function getPrimaryCategories(text) {
|
||||
const lower = text.toLowerCase();
|
||||
const cats = [];
|
||||
for (const [name, concepts] of Object.entries(CATEGORY_GROUPS)) {
|
||||
let hasMatch = false;
|
||||
for (const concept of concepts) {
|
||||
if (lower.includes(normalise(concept))) { hasMatch = true; break; }
|
||||
}
|
||||
if (hasMatch) cats.push(name);
|
||||
}
|
||||
return cats;
|
||||
}
|
||||
|
||||
/* ── Support categories a question draws from ─*/
|
||||
|
||||
function getSupportCategories(text) {
|
||||
const lower = text.toLowerCase();
|
||||
const cats = [];
|
||||
for (const [name, concepts] of Object.entries(SUPPORT_MAP)) {
|
||||
let hasMatch = false;
|
||||
for (const concept of concepts) {
|
||||
if (lower.includes(normalise(concept))) { hasMatch = true; break; }
|
||||
}
|
||||
if (hasMatch) cats.push(name);
|
||||
}
|
||||
return cats;
|
||||
}
|
||||
|
||||
/* ── Rule 1: Test a deciding condition directly ─*/
|
||||
|
||||
function testsCondition(conditions, primaryCategories, text) {
|
||||
let bestMatch = null;
|
||||
let bestScore = -1;
|
||||
let bestCoverage = 0;
|
||||
|
||||
for (const cond of conditions) {
|
||||
const condCats = getConditionCategories(cond);
|
||||
if (condCats.length === 0) continue;
|
||||
|
||||
let score = 0;
|
||||
let coverage = 0;
|
||||
|
||||
for (const pCat of primaryCategories) {
|
||||
if (!condCats.includes(pCat)) continue;
|
||||
|
||||
const group = CATEGORY_GROUPS[pCat];
|
||||
const qLower = text.toLowerCase();
|
||||
const cLower = cond.toLowerCase();
|
||||
|
||||
// Count distinct concepts from this category that appear in question or condition
|
||||
let distinctConcepts = 0;
|
||||
for (const c of group) {
|
||||
const normC = normalise(c);
|
||||
if (qLower.includes(normC) || cLower.includes(normC)) {
|
||||
distinctConcepts++;
|
||||
}
|
||||
}
|
||||
|
||||
score += Math.min(distinctConcepts, group.length);
|
||||
if (distinctConcepts > coverage) coverage = distinctConcepts;
|
||||
}
|
||||
|
||||
// Update: strict better score wins. Tied score: more concept coverage wins.
|
||||
if (score > bestScore || (score === bestScore && coverage > bestCoverage)) {
|
||||
bestMatch = cond;
|
||||
bestScore = score;
|
||||
bestCoverage = coverage;
|
||||
}
|
||||
}
|
||||
|
||||
return bestMatch;
|
||||
}
|
||||
|
||||
/* ── Rule 2: Add supporting evidence to a condition ─*/
|
||||
|
||||
function supportsCondition(conditions, supportCategories, primaryCategories, text) {
|
||||
let bestMatch = null;
|
||||
let bestScore = -1;
|
||||
|
||||
for (const cond of conditions) {
|
||||
const condCats = getConditionCategories(cond);
|
||||
|
||||
let score = 0;
|
||||
for (const sCat of supportCategories) {
|
||||
// Direct primary concept hits in this category's group
|
||||
const directHits = countPrimaryHits(text, sCat);
|
||||
|
||||
// Support-only concept hits
|
||||
let supportOnlyHits = 0;
|
||||
for (const c of SUPPORT_MAP[sCat] || []) {
|
||||
if (text.toLowerCase().includes(normalise(c))) supportOnlyHits++;
|
||||
}
|
||||
|
||||
score += directHits * 0.5 + supportOnlyHits * 0.3;
|
||||
}
|
||||
|
||||
if (score > bestScore) {
|
||||
bestMatch = cond;
|
||||
bestScore = score;
|
||||
}
|
||||
}
|
||||
|
||||
return bestMatch;
|
||||
}
|
||||
|
||||
function countPrimaryHits(text, categoryName) {
|
||||
const group = CATEGORY_GROUPS[categoryName];
|
||||
if (!group) return 0;
|
||||
const qLower = text.toLowerCase();
|
||||
let count = 0;
|
||||
for (const c of group) {
|
||||
if (qLower.includes(normalise(c))) count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
/* ── Core classification function ─────────────────────────── */
|
||||
|
||||
export function assessQuestionAgainstDecisionConditions(input) {
|
||||
const { decisionTarget, decisionConditions, unknown: node, graph } = input || {};
|
||||
|
||||
if (!decisionTarget || !node || typeof node.kind !== "string") {
|
||||
return { relevance: "cannot_determine", reason: "missing_input" };
|
||||
}
|
||||
if (!decisionConditions || !Array.isArray(decisionConditions) || decisionConditions.length === 0) {
|
||||
return { relevance: "cannot_determine", reason: "missing_or_empty_conditions" };
|
||||
}
|
||||
|
||||
const text = `${node?.label || ""} ${node?.description || ""}`.trim();
|
||||
if (!text) {
|
||||
return { relevance: "cannot_determine", reason: "empty_node_text" };
|
||||
}
|
||||
|
||||
const decisionText = normalise(decisionTarget);
|
||||
if (!decisionText) {
|
||||
return { relevance: "cannot_determine", reason: "empty_decision_target" };
|
||||
}
|
||||
|
||||
const validConditions = decisionConditions.filter((c) => c && normalise(c).length > 0);
|
||||
if (validConditions.length === 0) {
|
||||
return { relevance: "cannot_determine", reason: "all_conditions_empty" };
|
||||
}
|
||||
|
||||
const unknownNormalized = normalise(text);
|
||||
|
||||
const dtWords = new Set(decisionText.split(/\s+/));
|
||||
if (dtWords.size < 3) {
|
||||
return { relevance: "cannot_determine", reason: "decision_target_too_short" };
|
||||
}
|
||||
|
||||
const primaryCategories = getPrimaryCategories(unknownNormalized);
|
||||
const supportCategories = getSupportCategories(unknownNormalized);
|
||||
|
||||
// No connection to any condition
|
||||
if (primaryCategories.length === 0 && supportCategories.length === 0) {
|
||||
return { relevance: "outside_decision_conditions", reason: "question does not relate to any stated decision condition" };
|
||||
}
|
||||
|
||||
// Rule 1: Direct test of a deciding condition — requires primary category matches
|
||||
if (primaryCategories.length > 0) {
|
||||
const matchedDirect = testsCondition(validConditions, primaryCategories, unknownNormalized);
|
||||
if (matchedDirect) {
|
||||
return {
|
||||
relevance: "tests_deciding_condition",
|
||||
matchedCondition: matchedDirect,
|
||||
reason: `question directly tests a condition required for the decision: "${matchedDirect}"`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Rule 2: Supports a condition via indirect match
|
||||
const matchedSupport = supportsCondition(validConditions, supportCategories, primaryCategories, unknownNormalized);
|
||||
if (matchedSupport) {
|
||||
return {
|
||||
relevance: "adds_supporting_evidence",
|
||||
matchedCondition: matchedSupport,
|
||||
reason: `question provides evidence related to a decision condition rather than testing it directly: "${matchedSupport}"`,
|
||||
};
|
||||
}
|
||||
|
||||
// Rule 3: Outside (safety net)
|
||||
return { relevance: "outside_decision_conditions", reason: "question does not clearly connect to any stated decision condition" };
|
||||
}
|
||||
@@ -0,0 +1,520 @@
|
||||
/**
|
||||
* Experiment 22 — Tests for assessQuestionAgainstDecisionConditions.
|
||||
*
|
||||
* Validates the new classifier against explicit decision conditions.
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { assessQuestionAgainstDecisionConditions } from "@/lib/graph/question-decision-conditions.js";
|
||||
import { assessQuestionRelevanceToDecision } from "@/lib/graph/question-decision-relevance.js";
|
||||
import { buildScenarioFixture } from "@/lib/mocks/scenarios.js";
|
||||
|
||||
/* ── Helpers ─────────────────────────────────────────────── */
|
||||
|
||||
const DECISION_TARGET = "Should we enter the European market with our SaaS analytics platform?";
|
||||
|
||||
const CONDITIONS = [
|
||||
"Credible customer demand exists in Europe",
|
||||
"European compliance is achievable",
|
||||
"The expected market value justifies the cost of entry",
|
||||
"The product offers sufficient competitive differentiation",
|
||||
];
|
||||
|
||||
function makeUnknown(id, label) {
|
||||
return {
|
||||
id,
|
||||
label: label || `Unknown ${id}`,
|
||||
description: label || `Unknown ${id}`,
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "low",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
childIds: [],
|
||||
};
|
||||
}
|
||||
|
||||
function makeGraphFor(nodes) {
|
||||
return {
|
||||
nodes,
|
||||
edges: [],
|
||||
resolvedNodeIds: [],
|
||||
activeUnknownNodeId: null,
|
||||
centralStatement: "Test",
|
||||
currentSummary: "Test",
|
||||
reasoningState: null,
|
||||
};
|
||||
}
|
||||
|
||||
/* ── tests_deciding_condition ─────────────────────────────── */
|
||||
|
||||
describe("assessQuestionAgainstDecisionConditions — tests_deciding_condition", () => {
|
||||
it("classifies a demand question as tests_deciding_condition", () => {
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
decisionConditions: CONDITIONS,
|
||||
unknown: makeUnknown("d1", "Whether there is genuine demand for our category in Europe"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("tests_deciding_condition");
|
||||
expect(result.matchedCondition).toContain("demand");
|
||||
});
|
||||
|
||||
it("classifies a compliance question as tests_deciding_condition", () => {
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
decisionConditions: CONDITIONS,
|
||||
unknown: makeUnknown("d2", "Whether our product is suitable for European compliance requirements"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("tests_deciding_condition");
|
||||
expect(result.matchedCondition).toContain("compliance");
|
||||
});
|
||||
|
||||
it("classifies a competitive differentiation question as tests_deciding_condition", () => {
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
decisionConditions: CONDITIONS,
|
||||
unknown: makeUnknown("d3", "Whether we have competitive differentiation against existing European players"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("tests_deciding_condition");
|
||||
expect(result.matchedCondition).toContain("differentiation");
|
||||
});
|
||||
|
||||
it("classifies a value justification question as tests_deciding_condition", () => {
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
decisionConditions: CONDITIONS,
|
||||
unknown: makeUnknown("d4", "Whether the expected market value justifies the compliance investment"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("tests_deciding_condition");
|
||||
expect(result.matchedCondition).toContain("value");
|
||||
});
|
||||
});
|
||||
|
||||
/* ── adds_supporting_evidence ────────────────────────────── */
|
||||
|
||||
describe("assessQuestionAgainstDecisionConditions — adds_supporting_evidence", () => {
|
||||
it("classifies a customer segment question as adds_supporting_evidence", () => {
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
decisionConditions: CONDITIONS,
|
||||
unknown: makeUnknown("s1", "Which European countries should we target first"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("adds_supporting_evidence");
|
||||
});
|
||||
|
||||
it("classifies an investment detail question as adds_supporting_evidence", () => {
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
decisionConditions: CONDITIONS,
|
||||
unknown: makeUnknown("s2", "How should we structure the European subsidiary"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("adds_supporting_evidence");
|
||||
});
|
||||
});
|
||||
|
||||
/* ── outside_decision_conditions ─────────────────────────── */
|
||||
|
||||
describe("assessQuestionAgainstDecisionConditions — outside_decision_conditions", () => {
|
||||
it("classifies a background question as outside_decision_conditions", () => {
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
decisionConditions: CONDITIONS,
|
||||
unknown: makeUnknown("o1", "What is the company's current US headquarters location"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("outside_decision_conditions");
|
||||
});
|
||||
|
||||
it("classifies an unrelated observation as outside_decision_conditions", () => {
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
decisionConditions: CONDITIONS,
|
||||
unknown: makeUnknown("o2", "Our CEO's previous experience in Asian markets"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("outside_decision_conditions");
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Multiple condition matches ──────────────────────────── */
|
||||
|
||||
describe("assessQuestionAgainstDecisionConditions — multiple conditions matched", () => {
|
||||
it("returns the first matching condition when a question spans two conditions", () => {
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
decisionConditions: CONDITIONS,
|
||||
unknown: makeUnknown("m1", "What market demand and compliance requirements exist in Europe"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("tests_deciding_condition");
|
||||
// First condition that matches should be returned
|
||||
expect(result.matchedCondition).toContain("demand");
|
||||
});
|
||||
|
||||
it("treats tests_deciding_condition as higher priority than adds_supporting_evidence", () => {
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
decisionConditions: CONDITIONS,
|
||||
unknown: makeUnknown("m2", "Does our platform meet European data regulations"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("tests_deciding_condition");
|
||||
});
|
||||
});
|
||||
|
||||
/* ── cannot_determine — missing input ────────────────────── */
|
||||
|
||||
describe("assessQuestionAgainstDecisionConditions — cannot_determine (missing input)", () => {
|
||||
it("returns cannot_determine when decisionTarget is missing", () => {
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionConditions: CONDITIONS,
|
||||
unknown: makeUnknown("mi1", "Whether to enter Europe"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("returns cannot_determine when decisionTarget is empty string", () => {
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: "",
|
||||
decisionConditions: CONDITIONS,
|
||||
unknown: makeUnknown("mi2", "Whether to enter Europe"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("returns cannot_determine when decisionTarget is too short", () => {
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: "Europe",
|
||||
decisionConditions: CONDITIONS,
|
||||
unknown: makeUnknown("mi3", "Whether to enter"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("returns cannot_determine when unknown is missing", () => {
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
decisionConditions: CONDITIONS,
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("returns cannot_determine when unknown has empty label and description", () => {
|
||||
const node = makeUnknown("mi4", "");
|
||||
node.label = "";
|
||||
node.description = "";
|
||||
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
decisionConditions: CONDITIONS,
|
||||
unknown: node,
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("returns cannot_determine when no input object is provided", () => {
|
||||
const result = assessQuestionAgainstDecisionConditions(null);
|
||||
expect(result.relevance).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("returns cannot_determine when unknown node has no kind property", () => {
|
||||
const node = makeUnknown("mi5", "test");
|
||||
delete node.kind;
|
||||
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
decisionConditions: CONDITIONS,
|
||||
unknown: node,
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("cannot_determine");
|
||||
});
|
||||
});
|
||||
|
||||
/* ── cannot_determine — missing / empty conditions ──────── */
|
||||
|
||||
describe("assessQuestionAgainstDecisionConditions — cannot_determine (missing conditions)", () => {
|
||||
it("returns cannot_determine when decisionConditions is missing", () => {
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
unknown: makeUnknown("mc1", "Whether to enter Europe"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("returns cannot_determine when decisionConditions is an empty array", () => {
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
decisionConditions: [],
|
||||
unknown: makeUnknown("mc2", "Whether to enter Europe"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("returns cannot_determine when all conditions are empty strings", () => {
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
decisionConditions: ["", " ", ""],
|
||||
unknown: makeUnknown("mc3", "Whether to enter Europe"),
|
||||
});
|
||||
|
||||
expect(result.relevance).toBe("cannot_determine");
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Deterministic output ─────────────────────────────────── */
|
||||
|
||||
describe("assessQuestionAgainstDecisionConditions — deterministic output", () => {
|
||||
it("returns identical results for identical inputs", () => {
|
||||
const node = makeUnknown("det1", "Whether there is genuine demand in Europe");
|
||||
const input = { decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: node };
|
||||
const r1 = assessQuestionAgainstDecisionConditions(input);
|
||||
const r2 = assessQuestionAgainstDecisionConditions(input);
|
||||
expect(r1).toEqual(r2);
|
||||
});
|
||||
|
||||
it("returns the same classification across multiple calls", () => {
|
||||
const results = [];
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget: DECISION_TARGET,
|
||||
decisionConditions: CONDITIONS,
|
||||
unknown: makeUnknown(`det2-${i}`, "Does our platform comply with GDPR"),
|
||||
});
|
||||
results.push(result.relevance);
|
||||
}
|
||||
expect(new Set(results).size).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Input immutability ──────────────────────────────────── */
|
||||
|
||||
describe("assessQuestionAgainstDecisionConditions — input immutability", () => {
|
||||
it("does not mutate the unknown node", () => {
|
||||
const node = makeUnknown("imm1", "Whether to proceed with European market entry");
|
||||
const snapshot = JSON.parse(JSON.stringify(node));
|
||||
assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: node });
|
||||
expect(node).toEqual(snapshot);
|
||||
});
|
||||
|
||||
it("does not mutate the graph (when provided)", () => {
|
||||
const node = makeUnknown("imm2", "test");
|
||||
const graph = makeGraphFor([node]);
|
||||
const originalNodesJSON = JSON.stringify(graph.nodes);
|
||||
assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: node, graph });
|
||||
expect(JSON.stringify(graph.nodes)).toBe(originalNodesJSON);
|
||||
});
|
||||
|
||||
it("does not mutate the decisionTarget string", () => {
|
||||
const dt = "Should we enter the European market with our SaaS analytics platform?";
|
||||
assessQuestionAgainstDecisionConditions({ decisionTarget: dt, decisionConditions: CONDITIONS, unknown: makeUnknown("imm3", "test") });
|
||||
expect(dt).toBe("Should we enter the European market with our SaaS analytics platform?");
|
||||
});
|
||||
|
||||
it("does not mutate the decisionConditions array", () => {
|
||||
const conditions = [...CONDITIONS];
|
||||
assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: conditions, unknown: makeUnknown("imm4", "test") });
|
||||
expect(JSON.stringify(conditions)).toBe(JSON.stringify(CONDITIONS));
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Complete long-investigation sequence ──────────────────── */
|
||||
|
||||
describe("assessQuestionAgainstDecisionConditions — long investigation sequence", () => {
|
||||
const scenarioName = "long";
|
||||
const turnCount = 5;
|
||||
|
||||
const allResults = [];
|
||||
|
||||
for (let t = 0; t < turnCount; t++) {
|
||||
const fixture = buildScenarioFixture(scenarioName, t);
|
||||
expect(fixture).not.toBeNull(`Turn ${t} should have a valid fixture`);
|
||||
|
||||
const decisionTarget = fixture.situationGraph.centralStatement;
|
||||
const resolvedIds = new Set(fixture.situationGraph.resolvedNodeIds || []);
|
||||
|
||||
for (const node of fixture.situationGraph.nodes) {
|
||||
if (node.kind !== "unknown") continue;
|
||||
if (resolvedIds.has(node.id)) continue;
|
||||
|
||||
const result = assessQuestionAgainstDecisionConditions({
|
||||
decisionTarget,
|
||||
decisionConditions: CONDITIONS,
|
||||
unknown: node,
|
||||
graph: fixture.situationGraph,
|
||||
});
|
||||
|
||||
allResults.push({ turn: t, nodeId: node.id, label: node.label, relevance: result.relevance, matchedCondition: result.matchedCondition, reason: result.reason });
|
||||
}
|
||||
}
|
||||
|
||||
it("every unresolved unknown across all turns receives a valid classification", () => {
|
||||
const validCategories = ["tests_deciding_condition", "adds_supporting_evidence", "outside_decision_conditions", "cannot_determine"];
|
||||
for (const r of allResults) {
|
||||
expect(validCategories).toContain(r.relevance);
|
||||
}
|
||||
});
|
||||
|
||||
it("every unresolved unknown in the long sequence classifies as tests_deciding_condition", () => {
|
||||
for (const r of allResults) {
|
||||
expect(r.relevance).toBe("tests_deciding_condition");
|
||||
}
|
||||
});
|
||||
|
||||
it("each unresolved unknown matches its corresponding decision condition", () => {
|
||||
const expectedMatches = [
|
||||
{ turn: 0, nodeId: "u-1", contains: "demand" },
|
||||
{ turn: 1, nodeId: "u-2", contains: "compliance" },
|
||||
{ turn: 2, nodeId: "u-3", contains: "value" },
|
||||
{ turn: 3, nodeId: "u-4", contains: "differentiation" },
|
||||
];
|
||||
for (const expected of expectedMatches) {
|
||||
const r = allResults.find((x) => x.turn === expected.turn && x.nodeId === expected.nodeId);
|
||||
expect(r).toBeDefined();
|
||||
expect(r.relevance).toBe("tests_deciding_condition");
|
||||
expect(r.matchedCondition).toContain(expected.contains);
|
||||
}
|
||||
});
|
||||
|
||||
it("the sequence runs deterministically and does not mutate inputs", () => {
|
||||
const node = makeUnknown("det-seq", "Whether there is genuine demand in Europe");
|
||||
const r1 = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: node });
|
||||
const r2 = assessQuestionAgainstDecisionConditions({ decisionTarget: DECISION_TARGET, decisionConditions: CONDITIONS, unknown: node });
|
||||
expect(r1).toEqual(r2);
|
||||
expect(JSON.stringify(node)).toBe(JSON.stringify(makeUnknown("det-seq", "Whether there is genuine demand in Europe")));
|
||||
});
|
||||
|
||||
it("category distribution across the full sequence", () => {
|
||||
const categories = {};
|
||||
for (const r of allResults) {
|
||||
categories[r.relevance] = (categories[r.relevance] || 0) + 1;
|
||||
}
|
||||
console.log("\n=== Experiment 22 — Classification Results ===");
|
||||
console.log(`Total unresolved unknowns classified: ${allResults.length}`);
|
||||
console.log("Category distribution:", JSON.stringify(categories, null, 2));
|
||||
expect(allResults.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("detailed per-turn classifications", () => {
|
||||
const byTurn = {};
|
||||
for (const r of allResults) {
|
||||
if (!byTurn[r.turn]) byTurn[r.turn] = [];
|
||||
byTurn[r.turn].push(r);
|
||||
}
|
||||
for (const [turn, results] of Object.entries(byTurn)) {
|
||||
console.log(`\n--- Turn ${turn} (${results.length} unresolved unknowns) ---`);
|
||||
for (const r of results) {
|
||||
const shortLabel = r.label?.length > 60 ? r.label.slice(0, 60) + "..." : r.label;
|
||||
console.log(` [${r.relevance}] ${shortLabel}`);
|
||||
}
|
||||
}
|
||||
expect(allResults.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("Turn 0 (market demand) is classified as tests_deciding_condition", () => {
|
||||
const turn0 = allResults.find((r) => r.turn === 0);
|
||||
expect(turn0).toBeDefined();
|
||||
expect(turn0.relevance).toBe("tests_deciding_condition");
|
||||
});
|
||||
|
||||
it("Turns 1-2 (compliance, cost) are classified as tests_deciding_condition", () => {
|
||||
const turns1to2 = allResults.filter((r) => r.turn >= 1 && r.turn <= 2);
|
||||
for (const r of turns1to2) {
|
||||
expect(r.relevance).toBe("tests_deciding_condition");
|
||||
}
|
||||
});
|
||||
|
||||
it("Turn 3 (differentiation) is classified as tests_deciding_condition", () => {
|
||||
const turn3 = allResults.find((r) => r.turn === 3);
|
||||
expect(turn3).toBeDefined();
|
||||
expect(turn3.relevance).toBe("tests_deciding_condition");
|
||||
});
|
||||
|
||||
it("competitive differentiation becomes a clearly decisive classification", () => {
|
||||
const differentiating = allResults.find((r) => r.nodeId === "u-4" && r.turn === 3);
|
||||
expect(differentiating).toBeDefined();
|
||||
expect(differentiating.relevance).toBe("tests_deciding_condition");
|
||||
expect(differentiating.matchedCondition).toContain("differentiation");
|
||||
});
|
||||
|
||||
it("all classified questions include a non-empty reason", () => {
|
||||
for (const r of allResults) {
|
||||
if (r.relevance !== "cannot_determine") {
|
||||
expect(typeof r.reason).toBe("string");
|
||||
expect(r.reason.length).toBeGreaterThan(0);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("every unresolved unknown has a matchedCondition when classified", () => {
|
||||
for (const r of allResults) {
|
||||
if (r.relevance !== "cannot_determine" && r.relevance !== "outside_decision_conditions") {
|
||||
expect(r.matchedCondition).toBeDefined();
|
||||
expect(typeof r.matchedCondition).toBe("string");
|
||||
expect(r.matchedCondition.length).toBeGreaterThan(0);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("the complete long-investigation sequence runs without error and produces results", () => {
|
||||
expect(allResults.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Comparison with Experiment 21 — clearer distinction ─ */
|
||||
|
||||
describe("assessQuestionAgainstDecisionConditions — comparison with Experiment 21", () => {
|
||||
it("distinguishes between decisive and supporting questions (Experiment 22 vs 21)", () => {
|
||||
const conditions = CONDITIONS;
|
||||
|
||||
const decisions21 = [];
|
||||
const decisions22 = [];
|
||||
|
||||
for (let t = 0; t < 4; t++) {
|
||||
const fixture = buildScenarioFixture("long", t);
|
||||
if (!fixture) continue;
|
||||
const decisionTarget = fixture.situationGraph.centralStatement;
|
||||
|
||||
for (const node of fixture.situationGraph.nodes) {
|
||||
if (node.kind !== "unknown") continue;
|
||||
const resolvedIds = new Set(fixture.situationGraph.resolvedNodeIds || []);
|
||||
if (resolvedIds.has(node.id)) continue;
|
||||
|
||||
// Experiment 21 classification
|
||||
const r21 = assessQuestionRelevanceToDecision({ decisionTarget, unknown: node });
|
||||
decisions21.push({ turn: t, label: node.label, relevance: r21.relevance });
|
||||
|
||||
// Experiment 22 classification
|
||||
const r22 = assessQuestionAgainstDecisionConditions({ decisionTarget, decisionConditions: conditions, unknown: node });
|
||||
decisions22.push({ turn: t, label: node.label, relevance: r22.relevance });
|
||||
}
|
||||
}
|
||||
|
||||
const distinct21 = new Set(decisions21.map((r) => r.relevance));
|
||||
const distinct22 = new Set(decisions22.map((r) => r.relevance));
|
||||
|
||||
// Experiment 22 should have a clearer distinction than 21
|
||||
expect(distinct21.size).toBeGreaterThanOrEqual(1);
|
||||
expect(distinct22.size).toBeGreaterThanOrEqual(1);
|
||||
|
||||
// Verify at least some questions moved to tests_deciding_condition in Exp 22
|
||||
const decisiveCount = decisions22.filter((r) => r.relevance === "tests_deciding_condition").length;
|
||||
expect(decisiveCount).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user