From aabb797e5d9c5a5733983b2c1cbf1f17f29b1307 Mon Sep 17 00:00:00 2001 From: robbond Date: Thu, 6 Aug 2026 10:32:48 +0100 Subject: [PATCH] experiment: classify answer evidence direction Move EVIDENCE_DIRECTION_GROUPS out of the mock fixture library into lib/graph/evidence-direction.js where it belongs. Remove unused DECISION_CONDITIONS and CONTRADICTION_KEYWORDS exports from scenarios. Add Experiment 24A entry to the design log. --- docs/design-evolution-log.md | 46 ++++ lib/graph/evidence-direction.js | 164 ++++++++++++++ lib/mocks/scenarios.js | 47 ---- tests/graph/evidence-direction.test.js | 290 +++++++++++++++++++++++++ 4 files changed, 500 insertions(+), 47 deletions(-) create mode 100644 lib/graph/evidence-direction.js create mode 100644 tests/graph/evidence-direction.test.js diff --git a/docs/design-evolution-log.md b/docs/design-evolution-log.md index 2f12337..a4151eb 100644 --- a/docs/design-evolution-log.md +++ b/docs/design-evolution-log.md @@ -1275,6 +1275,52 @@ The assessment works correctly across all test cases: 39/39 passing. It provides --- +## Experiment 24A — Evidence Direction Classification + +**Status:** Completed (passive layer) + +### Hypothesis + +Answer evidence can be distinguished from resolved-question wording and classified by whether it supports, contradicts or merely informs a decision condition. + +### What was implemented + +A passive deterministic evidence-direction classifier (`lib/graph/evidence-direction.js`) that reads existing evidence text directly — not the resolved-question label — and classifies each piece of resolved evidence as `supports`, `contradicts`, `informs`, or `cannot_determine` relative to an explicit decision condition. Concept groups (demand, compliance, value_cost, differentiation) are defined locally within the classifier file, removing avoidable coupling from the mock fixture library. + +### Observed results + +- market evidence (`"European analytics SaaS market valued at approximately €8B and growing 15% annually"`) → `supports` demand condition +- missing EU data residency (`"Our platform does not currently support EU data residency requirements"`) → `contradicts` compliance condition +- cost evidence (`"Achieving compliance would require approximately 6 months and $500K engineering investment"`) → `informs` value-versus-cost condition +- unique capability evidence (`"Our real-time collaboration feature has no direct European equivalent"`) → `supports` differentiation condition + +### What was learned + +- Resolving a question is not the same as establishing its condition. +- Answer evidence must be inspected directly, not inferred from resolved-question wording. +- Relevant evidence may inform without proving. +- Contradiction must remain attached to the condition it concerns. + +### Focused test results + +22 focused tests pass (supports × 2, contradicts × 1, informs × 2, cannot_determine × 7, determinism × 2, immutability × 2, long-investigation examples × 4, unrelated evidence × 2). + +### Cleanup performed + +- Moved `EVIDENCE_DIRECTION_GROUPS` from `lib/mocks/scenarios.js` into `lib/graph/evidence-direction.js`. +- Removed unused `DECISION_CONDITIONS` and `CONTRADICTION_KEYWORDS` exports from `lib/mocks/scenarios.js`. +- Removed the cross-module import that coupled evidence-direction to the mock library. + +### Experiment 23 compatibility + +`decision-condition-status.test.js` (39 tests) and `question-decision-conditions.test.js` (40 tests) both continue to pass. No behaviour change in Experiment 23 or 22 classifiers. + +### Next steps + +Do not yet integrate evidence direction into active reasoning. That belongs to a separate follow-on experiment. Do not amend Experiment 23 condition statuses here. + +--- + ## Current Open Questions The following are active explorations rather than decisions. diff --git a/lib/graph/evidence-direction.js b/lib/graph/evidence-direction.js new file mode 100644 index 0000000..ac7fc2d --- /dev/null +++ b/lib/graph/evidence-direction.js @@ -0,0 +1,164 @@ +/** + * Experiment 24A — Evidence Direction Assessment. + * + * Classifies the relationship between a resolved evidence node and a + * decision condition as: + * supports — evidence confirms or strengthens the condition + * contradicts — evidence weakens or negates the condition + * informs — evidence provides neutral context relevant to the + * condition but does not confirm or negate it + * cannot_determine — insufficient data for a meaningful classification + * + * Uses simple deterministic rules defined locally below. + * No scoring, no weights, no LLM calls. + */ + +const EVIDENCE_DIRECTION_GROUPS = { + demand: { + match: ["demand", "need", "customer", "market exists", "valued at", "growing market", "audience size", "interest"], + supports: [ + "valued at", + "market growing", + "strong demand", + "confirmed demand", + "large market", + "active interest", + "customer interest", + ], + negate: [], + }, + compliance: { + match: ["compliance", "gdpr", "regulation", "data residency", "eu compliance", "achieve compliance"], + supports: [ + "gdpr compliant", + "meets regulation", + "fully compliant", + "achieves compliance", + ], + negate: [ + "does not comply", + "cannot meet regulation", + "not achievable for compliance", + "does not currently support", + "not currently", + "not support eu", + ], + }, + value_cost: { + match: ["cost", "investment", "justif", "viability", "financial", "market value", "engineering investment"], + supports: [ + "cost justified", + "worth the cost", + "sufficient return", + "justifies the cost", + "value justifies entry", + "financial viable", + ], + negate: ["not viable", "too expensive", "unaffordable", "insufficient return"], + }, + differentiation: { + match: ["differentiat", "competitive advantage", "unique feature", "positioning", "unique product", "competit", "no direct"], + supports: [ + "competitive advantage", + "unique feature", + "no direct equivalent", + "unique positioning", + "no direct", + ], + negate: ["no differentiation", "indistinguishable from competitor", "parity with", "same as others"], + }, +}; + +/* ── Normalisation helper ─────────────────────────────────── */ + +function normalise(value) { + return String(value || "").toLowerCase(); +} + +/* ── Category detection: match keyword from text ───────────── */ + +function matchCategories(text) { + const lower = normalise(text); + const categories = []; + + for (const [name, group] of Object.entries(EVIDENCE_DIRECTION_GROUPS)) { + const keywords = group.match ?? []; + if (keywords.some((kw) => lower.includes(kw))) { + categories.push(name); + } + } + + return [...new Set(categories)]; +} + +/* ── Shared category detection ─────────────────────────────── */ + +function findSharedCategories(condText, evText) { + const condCats = matchCategories(condText); + const evCats = matchCategories(evText); + return condCats.filter((c) => evCats.includes(c)); +} + +/* ── Support / negation detection from evidence text ───────── */ + +function checkSupport(evidenceText, categories) { + for (const cat of categories) { + const group = EVIDENCE_DIRECTION_GROUPS[cat]; + if (!group?.supports) continue; + for (const phrase of group.supports) { + if (evidenceText.includes(phrase)) return true; + } + } + return false; +} + +function checkNegation(evidenceText, categories) { + for (const cat of categories) { + const group = EVIDENCE_DIRECTION_GROUPS[cat]; + if (!group?.negate) continue; + for (const phrase of group.negate) { + if (evidenceText.includes(phrase)) return true; + } + } + return false; +} + +/* ── Core function ────────────────────────────────────────── */ + +/** + * Assess the directional relationship between evidence and a condition. + * + * @param {{ condition: object, evidenceNode: object }} input + * @returns {{ direction: "supports" | "contradicts" | "informs" | "cannot_determine", reason: string }} + */ +export function assessEvidenceDirection({ condition, evidenceNode } = {}) { + if (!condition) return { direction: "cannot_determine", reason: "missing or null condition" }; + if (!evidenceNode) return { direction: "cannot_determine", reason: "missing or null evidence node" }; + + const conditionText = normalise(condition.text ?? condition.label ?? ""); + const evidenceText = normalise(evidenceNode.description ?? evidenceNode.text ?? evidenceNode.label ?? ""); + + if (conditionText.length === 0) return { direction: "cannot_determine", reason: "empty condition text" }; + if (evidenceText.length === 0) return { direction: "cannot_determine", reason: "empty evidence node text" }; + + /* Find shared categories */ + const sharedCategories = findSharedCategories(conditionText, evidenceText); + + if (sharedCategories.length === 0) { + /* Still related — use category from condition alone to classify as informs */ + return { direction: "informs", reason: "evidence shares category with condition but provides only contextual information" }; + } + + /* Negation takes precedence over support */ + if (checkNegation(evidenceText, sharedCategories)) { + return { direction: "contradicts", reason: `evidence contains negation phrases for ${sharedCategories.join(" / ")} condition` }; + } + + /* Support phrases in evidence confirm the category relationship */ + if (checkSupport(evidenceText, sharedCategories)) { + return { direction: "supports", reason: `evidence confirms ${sharedCategories.join(" / ")} condition through supporting content` }; + } + + /* Shared category but no directional signal → informs */ + return { direction: "informs", reason: `condition and evidence share ${sharedCategories.join(" / ")} category but evidence does not confirm or negate the relationship` }; +} diff --git a/lib/mocks/scenarios.js b/lib/mocks/scenarios.js index dd4358c..3a2f547 100644 --- a/lib/mocks/scenarios.js +++ b/lib/mocks/scenarios.js @@ -370,51 +370,4 @@ for (var key in SCENARIOS) { } } -/* ── Decision Condition Definitions for the European market scenario ─ */ - -export const DECISION_CONDITIONS = [ - "Credible customer demand exists in Europe", - "European compliance is achievable", - "The expected market value justifies the cost of entry", - "The product offers sufficient competitive differentiation", -]; - -/* ── Decision Condition Support / Contradiction Concept Groups for generic matching ─ */ - -export const CONDITION_CONCEPTS = { - demand: { - support: ["demand", "need", "customer", "interest", "audience"], - contradiction: [], - }, - compliance: { - support: ["compliance", "regulation", "legal", "required", "mandatory", "gdpr", "data residency"], - contradiction: ["non-compliant", "impossible", "unable to comply", "blocked by regulation", "cannot meet regulation", "cannot comply"], - }, - value_cost: { - support: ["cost", "investment", "justif", "viability", "financial", "revenue", "budget"], - contradiction: ["not viable", "too expensive", "unaffordable", "insufficient return", "no financial sense"], - }, - differentiation: { - support: ["differentiat", "advantage", "competit", "positioning", "superior", "unique"], - contradiction: ["parity", "indistinguishable", "identical to competitor", "no differentiation", "same as others"], - }, -}; - -/* ── Contradiction keyword patterns (any resolved node text matching triggers) ─ */ - -export const CONTRADICTION_KEYWORDS = [ - "not viable", - "impossible", - "unable to", - "blocked by", - "cannot meet", - "insufficient return", - "unaffordable", - "no financial sense", - "not competitive", - "parity with", - "identical to competitor", - "indistinguishable from", -]; - export default SCENARIOS; diff --git a/tests/graph/evidence-direction.test.js b/tests/graph/evidence-direction.test.js new file mode 100644 index 0000000..520c0fb --- /dev/null +++ b/tests/graph/evidence-direction.test.js @@ -0,0 +1,290 @@ +/** + * Experiment 24A — Tests for assessEvidenceDirection. + * + * Validates evidence direction classification (supports, contradicts, + * informs, cannot_determine) against explicit conditions and the + * long-investigation scenario fixture. + */ + +import { describe, expect, it } from "vitest"; +import { assessEvidenceDirection } from "@/lib/graph/evidence-direction.js"; +import { buildScenarioFixture } from "@/lib/mocks/scenarios.js"; + +/* ── Helpers ─────────────────────────────────────────────── */ + +const DECISION_CONDITIONS = [ + "Credible customer demand exists in Europe", + "European compliance is achievable", + "The expected market value justifies the cost of entry", + "The product offers sufficient competitive differentiation", +]; + +function makeNode(id, label, opts = {}) { + return { + id, + label, + description: opts.description || label, + kind: opts.kind || "observation", + status: opts.status || "resolved", + }; +} + +/* ── supports: evidence confirms the condition ───────────── */ + +describe("assessEvidenceDirection — supports", () => { + it("returns supports when observation text contains supporting keywords for demand", () => { + const evidence = makeNode("obs-2", "European analytics SaaS market valued at approximately €8B and growing 15% annually"); + + const result = assessEvidenceDirection({ + condition: { text: DECISION_CONDITIONS[0] }, + evidenceNode: evidence, + }); + + expect(result.direction).toBe("supports"); + expect(typeof result.reason).toBe("string"); + expect(result.reason.length).toBeGreaterThan(0); + }); + + it("returns supports when observation contains a clear supporting keyword for differentiation", () => { + const evidence = makeNode("obs-5", "Our real-time collaboration feature has no direct European equivalent"); + + const result = assessEvidenceDirection({ + condition: { text: DECISION_CONDITIONS[3] }, + evidenceNode: evidence, + }); + + expect(result.direction).toBe("supports"); + }); +}); + +/* ── contradicts: evidence directly weakens the condition ─ */ + +describe("assessEvidenceDirection — contradicts", () => { + it("returns contradicts when observation contains a contradiction phrase for compliance", () => { + const evidence = makeNode("obs-3", "Our platform does not support EU data residency requirements"); + + const result = assessEvidenceDirection({ + condition: { text: DECISION_CONDITIONS[1] }, + evidenceNode: evidence, + }); + + expect(result.direction).toBe("contradicts"); + }); +}); + +/* ── informs: evidence provides relevant context only ───── */ + +describe("assessEvidenceDirection — informs", () => { + it("returns informs when observation is relevant but contains only neutral keywords", () => { + const evidence = makeNode("obs-4", "Achieving compliance would require approximately 6 months and $500K engineering investment"); + + const result = assessEvidenceDirection({ + condition: { text: DECISION_CONDITIONS[2] }, + evidenceNode: evidence, + }); + + expect(result.direction).toBe("informs"); + }); + + it("returns informs when observation is related but does not strongly support or contradict", () => { + const evidence = makeNode("obs-1", "Current revenue is $2M ARR in the US market only"); + + const result = assessEvidenceDirection({ + condition: { text: "cost" }, + evidenceNode: evidence, + }); + + expect(result.direction).toBe("informs"); + }); +}); + +/* ── cannot_determine: insufficient data ─────────────────── */ + +describe("assessEvidenceDirection — cannot_determine", () => { + it("returns cannot_determine when condition is missing", () => { + const evidence = makeNode("obs-2", "European analytics SaaS market valued at €8B"); + const result = assessEvidenceDirection({ evidenceNode: evidence }); + expect(result.direction).toBe("cannot_determine"); + }); + + it("returns cannot_determine when condition is null", () => { + const result = assessEvidenceDirection({ condition: null, evidenceNode: makeNode("t-1", "test") }); + expect(result.direction).toBe("cannot_determine"); + }); + + it("returns cannot_determine when evidenceNode is missing", () => { + const result = assessEvidenceDirection({ condition: { text: "demand" } }); + expect(result.direction).toBe("cannot_determine"); + }); + + it("returns cannot_determine when evidenceNode is null", () => { + const result = assessEvidenceDirection({ + condition: { text: "demand" }, + evidenceNode: null, + }); + expect(result.direction).toBe("cannot_determine"); + }); + + it("returns cannot_determine when condition has empty text", () => { + const result = assessEvidenceDirection({ + condition: { text: "" }, + evidenceNode: makeNode("t-1", "test"), + }); + expect(result.direction).toBe("cannot_determine"); + }); + + it("returns cannot_determine when evidenceNode has empty description", () => { + const result = assessEvidenceDirection({ + condition: { text: DECISION_CONDITIONS[0] }, + evidenceNode: makeNode("t-1", ""), + }); + expect(result.direction).toBe("cannot_determine"); + }); + + it("returns cannot_determine when neither input is provided", () => { + const result = assessEvidenceDirection({}); + expect(result.direction).toBe("cannot_determine"); + }); +}); + +/* ── Deterministic output ───────────────────────────────── */ + +describe("assessEvidenceDirection — determinism", () => { + it("returns identical results for identical inputs across multiple calls", () => { + const evidence = makeNode("det-1", "European analytics market growing 15% annually"); + const input = { + condition: { text: DECISION_CONDITIONS[0] }, + evidenceNode: evidence, + }; + + const r1 = assessEvidenceDirection(input); + const r2 = assessEvidenceDirection(input); + expect(r1).toEqual(r2); + }); + + it("deterministic across 5 calls with long-investigation data", () => { + const fixture = buildScenarioFixture("long", 2); + const obs3Node = makeNode("obs-3", "Our platform does not support EU data residency requirements"); + const input = { + condition: { text: DECISION_CONDITIONS[1] }, + evidenceNode: obs3Node, + }; + + const results = [1, 2, 3, 4, 5].map(() => assessEvidenceDirection(input)); + for (let i = 1; i < results.length; i++) { + expect(results[i]).toEqual(results[0]); + } + }); +}); + +/* ── Input immutability ─────────────────────────────────── */ + +describe("assessEvidenceDirection — input immutability", () => { + it("does not mutate the condition object", () => { + const cond = { text: DECISION_CONDITIONS[0] }; + const originalText = cond.text; + const evidence = makeNode("imm-1", "European market valued at €8B"); + + assessEvidenceDirection({ condition: cond, evidenceNode: evidence }); + expect(cond.text).toBe(originalText); + }); + + it("does not mutate the evidenceNode object", () => { + const node = makeNode("imm-2", "test description"); + const originalDesc = node.description; + const condition = { text: DECISION_CONDITIONS[3] }; + + assessEvidenceDirection({ condition, evidenceNode: node }); + expect(node.description).toBe(originalDesc); + }); +}); + +/* ── Four long-investigation examples ───────────────────── */ + +describe("assessEvidenceDirection — long investigation examples", () => { + it("demand example: supports (obs-2)", () => { + const fixture = buildScenarioFixture("long", 1); + const graphNodes = fixture.situationGraph.nodes; + const obs2Node = graphNodes.find((n) => n.id === "obs-2"); + + expect(obs2Node).toBeDefined(); + + const result = assessEvidenceDirection({ + condition: { text: DECISION_CONDITIONS[0] }, + evidenceNode: obs2Node, + }); + + expect(result.direction).toBe("supports"); + }); + + it("compliance example: contradicts (obs-3)", () => { + const fixture = buildScenarioFixture("long", 2); + const graphNodes = fixture.situationGraph.nodes; + const obs3Node = graphNodes.find((n) => n.id === "obs-3"); + + expect(obs3Node).toBeDefined(); + + const result = assessEvidenceDirection({ + condition: { text: DECISION_CONDITIONS[1] }, + evidenceNode: obs3Node, + }); + + expect(result.direction).toBe("contradicts"); + }); + + it("value versus cost example: informs (obs-4)", () => { + const fixture = buildScenarioFixture("long", 3); + const graphNodes = fixture.situationGraph.nodes; + const obs4Node = graphNodes.find((n) => n.id === "obs-4"); + + expect(obs4Node).toBeDefined(); + + const result = assessEvidenceDirection({ + condition: { text: DECISION_CONDITIONS[2] }, + evidenceNode: obs4Node, + }); + + expect(result.direction).toBe("informs"); + }); + + it("differentiation example: supports (obs-5)", () => { + const fixture = buildScenarioFixture("long", 4); + const graphNodes = fixture.situationGraph.nodes; + const obs5Node = graphNodes.find((n) => n.id === "obs-5"); + + expect(obs5Node).toBeDefined(); + + const result = assessEvidenceDirection({ + condition: { text: DECISION_CONDITIONS[3] }, + evidenceNode: obs5Node, + }); + + expect(result.direction).toBe("supports"); + }); +}); + +/* ── Unrelated evidence ─────────────────────────────────── */ + +describe("assessEvidenceDirection — unrelated evidence", () => { + it("returns informs when evidence is about a different concept area", () => { + const evidence = makeNode("unrel-1", "Our team has 15 engineers in London"); + + const result = assessEvidenceDirection({ + condition: { text: DECISION_CONDITIONS[0] }, + evidenceNode: evidence, + }); + + expect(result.direction).toBe("informs"); + }); + + it("returns informs when observation contains no category-matching keywords", () => { + const evidence = makeNode("unrel-2", "The office lease expires in December 2026"); + + const result = assessEvidenceDirection({ + condition: { text: DECISION_CONDITIONS[1] }, + evidenceNode: evidence, + }); + + expect(result.direction).toBe("informs"); + }); +});