experiment: classify answer evidence direction

Move EVIDENCE_DIRECTION_GROUPS out of the mock fixture library into
lib/graph/evidence-direction.js where it belongs. Remove unused
DECISION_CONDITIONS and CONTRADICTION_KEYWORDS exports from scenarios.

Add Experiment 24A entry to the design log.
This commit is contained in:
2026-08-06 10:32:48 +01:00
parent 3119635211
commit aabb797e5d
4 changed files with 500 additions and 47 deletions
+46
View File
@@ -1275,6 +1275,52 @@ The assessment works correctly across all test cases: 39/39 passing. It provides
---
## Experiment 24A — Evidence Direction Classification
**Status:** Completed (passive layer)
### Hypothesis
Answer evidence can be distinguished from resolved-question wording and classified by whether it supports, contradicts or merely informs a decision condition.
### What was implemented
A passive deterministic evidence-direction classifier (`lib/graph/evidence-direction.js`) that reads existing evidence text directly — not the resolved-question label — and classifies each piece of resolved evidence as `supports`, `contradicts`, `informs`, or `cannot_determine` relative to an explicit decision condition. Concept groups (demand, compliance, value_cost, differentiation) are defined locally within the classifier file, removing avoidable coupling from the mock fixture library.
### Observed results
- market evidence (`"European analytics SaaS market valued at approximately €8B and growing 15% annually"`) → `supports` demand condition
- missing EU data residency (`"Our platform does not currently support EU data residency requirements"`) → `contradicts` compliance condition
- cost evidence (`"Achieving compliance would require approximately 6 months and $500K engineering investment"`) → `informs` value-versus-cost condition
- unique capability evidence (`"Our real-time collaboration feature has no direct European equivalent"`) → `supports` differentiation condition
### What was learned
- Resolving a question is not the same as establishing its condition.
- Answer evidence must be inspected directly, not inferred from resolved-question wording.
- Relevant evidence may inform without proving.
- Contradiction must remain attached to the condition it concerns.
### Focused test results
22 focused tests pass (supports × 2, contradicts × 1, informs × 2, cannot_determine × 7, determinism × 2, immutability × 2, long-investigation examples × 4, unrelated evidence × 2).
### Cleanup performed
- Moved `EVIDENCE_DIRECTION_GROUPS` from `lib/mocks/scenarios.js` into `lib/graph/evidence-direction.js`.
- Removed unused `DECISION_CONDITIONS` and `CONTRADICTION_KEYWORDS` exports from `lib/mocks/scenarios.js`.
- Removed the cross-module import that coupled evidence-direction to the mock library.
### Experiment 23 compatibility
`decision-condition-status.test.js` (39 tests) and `question-decision-conditions.test.js` (40 tests) both continue to pass. No behaviour change in Experiment 23 or 22 classifiers.
### Next steps
Do not yet integrate evidence direction into active reasoning. That belongs to a separate follow-on experiment. Do not amend Experiment 23 condition statuses here.
---
## Current Open Questions
The following are active explorations rather than decisions.
+164
View File
@@ -0,0 +1,164 @@
/**
* Experiment 24A Evidence Direction Assessment.
*
* Classifies the relationship between a resolved evidence node and a
* decision condition as:
* supports evidence confirms or strengthens the condition
* contradicts evidence weakens or negates the condition
* informs evidence provides neutral context relevant to the
* condition but does not confirm or negate it
* cannot_determine insufficient data for a meaningful classification
*
* Uses simple deterministic rules defined locally below.
* No scoring, no weights, no LLM calls.
*/
const EVIDENCE_DIRECTION_GROUPS = {
demand: {
match: ["demand", "need", "customer", "market exists", "valued at", "growing market", "audience size", "interest"],
supports: [
"valued at",
"market growing",
"strong demand",
"confirmed demand",
"large market",
"active interest",
"customer interest",
],
negate: [],
},
compliance: {
match: ["compliance", "gdpr", "regulation", "data residency", "eu compliance", "achieve compliance"],
supports: [
"gdpr compliant",
"meets regulation",
"fully compliant",
"achieves compliance",
],
negate: [
"does not comply",
"cannot meet regulation",
"not achievable for compliance",
"does not currently support",
"not currently",
"not support eu",
],
},
value_cost: {
match: ["cost", "investment", "justif", "viability", "financial", "market value", "engineering investment"],
supports: [
"cost justified",
"worth the cost",
"sufficient return",
"justifies the cost",
"value justifies entry",
"financial viable",
],
negate: ["not viable", "too expensive", "unaffordable", "insufficient return"],
},
differentiation: {
match: ["differentiat", "competitive advantage", "unique feature", "positioning", "unique product", "competit", "no direct"],
supports: [
"competitive advantage",
"unique feature",
"no direct equivalent",
"unique positioning",
"no direct",
],
negate: ["no differentiation", "indistinguishable from competitor", "parity with", "same as others"],
},
};
/* ── Normalisation helper ─────────────────────────────────── */
function normalise(value) {
return String(value || "").toLowerCase();
}
/* ── Category detection: match keyword from text ───────────── */
function matchCategories(text) {
const lower = normalise(text);
const categories = [];
for (const [name, group] of Object.entries(EVIDENCE_DIRECTION_GROUPS)) {
const keywords = group.match ?? [];
if (keywords.some((kw) => lower.includes(kw))) {
categories.push(name);
}
}
return [...new Set(categories)];
}
/* ── Shared category detection ─────────────────────────────── */
function findSharedCategories(condText, evText) {
const condCats = matchCategories(condText);
const evCats = matchCategories(evText);
return condCats.filter((c) => evCats.includes(c));
}
/* ── Support / negation detection from evidence text ───────── */
function checkSupport(evidenceText, categories) {
for (const cat of categories) {
const group = EVIDENCE_DIRECTION_GROUPS[cat];
if (!group?.supports) continue;
for (const phrase of group.supports) {
if (evidenceText.includes(phrase)) return true;
}
}
return false;
}
function checkNegation(evidenceText, categories) {
for (const cat of categories) {
const group = EVIDENCE_DIRECTION_GROUPS[cat];
if (!group?.negate) continue;
for (const phrase of group.negate) {
if (evidenceText.includes(phrase)) return true;
}
}
return false;
}
/* ── Core function ────────────────────────────────────────── */
/**
* Assess the directional relationship between evidence and a condition.
*
* @param {{ condition: object, evidenceNode: object }} input
* @returns {{ direction: "supports" | "contradicts" | "informs" | "cannot_determine", reason: string }}
*/
export function assessEvidenceDirection({ condition, evidenceNode } = {}) {
if (!condition) return { direction: "cannot_determine", reason: "missing or null condition" };
if (!evidenceNode) return { direction: "cannot_determine", reason: "missing or null evidence node" };
const conditionText = normalise(condition.text ?? condition.label ?? "");
const evidenceText = normalise(evidenceNode.description ?? evidenceNode.text ?? evidenceNode.label ?? "");
if (conditionText.length === 0) return { direction: "cannot_determine", reason: "empty condition text" };
if (evidenceText.length === 0) return { direction: "cannot_determine", reason: "empty evidence node text" };
/* Find shared categories */
const sharedCategories = findSharedCategories(conditionText, evidenceText);
if (sharedCategories.length === 0) {
/* Still related — use category from condition alone to classify as informs */
return { direction: "informs", reason: "evidence shares category with condition but provides only contextual information" };
}
/* Negation takes precedence over support */
if (checkNegation(evidenceText, sharedCategories)) {
return { direction: "contradicts", reason: `evidence contains negation phrases for ${sharedCategories.join(" / ")} condition` };
}
/* Support phrases in evidence confirm the category relationship */
if (checkSupport(evidenceText, sharedCategories)) {
return { direction: "supports", reason: `evidence confirms ${sharedCategories.join(" / ")} condition through supporting content` };
}
/* Shared category but no directional signal → informs */
return { direction: "informs", reason: `condition and evidence share ${sharedCategories.join(" / ")} category but evidence does not confirm or negate the relationship` };
}
-47
View File
@@ -370,51 +370,4 @@ for (var key in SCENARIOS) {
}
}
/* ── Decision Condition Definitions for the European market scenario ─ */
export const DECISION_CONDITIONS = [
"Credible customer demand exists in Europe",
"European compliance is achievable",
"The expected market value justifies the cost of entry",
"The product offers sufficient competitive differentiation",
];
/* ── Decision Condition Support / Contradiction Concept Groups for generic matching ─ */
export const CONDITION_CONCEPTS = {
demand: {
support: ["demand", "need", "customer", "interest", "audience"],
contradiction: [],
},
compliance: {
support: ["compliance", "regulation", "legal", "required", "mandatory", "gdpr", "data residency"],
contradiction: ["non-compliant", "impossible", "unable to comply", "blocked by regulation", "cannot meet regulation", "cannot comply"],
},
value_cost: {
support: ["cost", "investment", "justif", "viability", "financial", "revenue", "budget"],
contradiction: ["not viable", "too expensive", "unaffordable", "insufficient return", "no financial sense"],
},
differentiation: {
support: ["differentiat", "advantage", "competit", "positioning", "superior", "unique"],
contradiction: ["parity", "indistinguishable", "identical to competitor", "no differentiation", "same as others"],
},
};
/* ── Contradiction keyword patterns (any resolved node text matching triggers) ─ */
export const CONTRADICTION_KEYWORDS = [
"not viable",
"impossible",
"unable to",
"blocked by",
"cannot meet",
"insufficient return",
"unaffordable",
"no financial sense",
"not competitive",
"parity with",
"identical to competitor",
"indistinguishable from",
];
export default SCENARIOS;
+290
View File
@@ -0,0 +1,290 @@
/**
* Experiment 24A Tests for assessEvidenceDirection.
*
* Validates evidence direction classification (supports, contradicts,
* informs, cannot_determine) against explicit conditions and the
* long-investigation scenario fixture.
*/
import { describe, expect, it } from "vitest";
import { assessEvidenceDirection } from "@/lib/graph/evidence-direction.js";
import { buildScenarioFixture } from "@/lib/mocks/scenarios.js";
/* ── Helpers ─────────────────────────────────────────────── */
const DECISION_CONDITIONS = [
"Credible customer demand exists in Europe",
"European compliance is achievable",
"The expected market value justifies the cost of entry",
"The product offers sufficient competitive differentiation",
];
function makeNode(id, label, opts = {}) {
return {
id,
label,
description: opts.description || label,
kind: opts.kind || "observation",
status: opts.status || "resolved",
};
}
/* ── supports: evidence confirms the condition ───────────── */
describe("assessEvidenceDirection — supports", () => {
it("returns supports when observation text contains supporting keywords for demand", () => {
const evidence = makeNode("obs-2", "European analytics SaaS market valued at approximately €8B and growing 15% annually");
const result = assessEvidenceDirection({
condition: { text: DECISION_CONDITIONS[0] },
evidenceNode: evidence,
});
expect(result.direction).toBe("supports");
expect(typeof result.reason).toBe("string");
expect(result.reason.length).toBeGreaterThan(0);
});
it("returns supports when observation contains a clear supporting keyword for differentiation", () => {
const evidence = makeNode("obs-5", "Our real-time collaboration feature has no direct European equivalent");
const result = assessEvidenceDirection({
condition: { text: DECISION_CONDITIONS[3] },
evidenceNode: evidence,
});
expect(result.direction).toBe("supports");
});
});
/* ── contradicts: evidence directly weakens the condition ─ */
describe("assessEvidenceDirection — contradicts", () => {
it("returns contradicts when observation contains a contradiction phrase for compliance", () => {
const evidence = makeNode("obs-3", "Our platform does not support EU data residency requirements");
const result = assessEvidenceDirection({
condition: { text: DECISION_CONDITIONS[1] },
evidenceNode: evidence,
});
expect(result.direction).toBe("contradicts");
});
});
/* ── informs: evidence provides relevant context only ───── */
describe("assessEvidenceDirection — informs", () => {
it("returns informs when observation is relevant but contains only neutral keywords", () => {
const evidence = makeNode("obs-4", "Achieving compliance would require approximately 6 months and $500K engineering investment");
const result = assessEvidenceDirection({
condition: { text: DECISION_CONDITIONS[2] },
evidenceNode: evidence,
});
expect(result.direction).toBe("informs");
});
it("returns informs when observation is related but does not strongly support or contradict", () => {
const evidence = makeNode("obs-1", "Current revenue is $2M ARR in the US market only");
const result = assessEvidenceDirection({
condition: { text: "cost" },
evidenceNode: evidence,
});
expect(result.direction).toBe("informs");
});
});
/* ── cannot_determine: insufficient data ─────────────────── */
describe("assessEvidenceDirection — cannot_determine", () => {
it("returns cannot_determine when condition is missing", () => {
const evidence = makeNode("obs-2", "European analytics SaaS market valued at €8B");
const result = assessEvidenceDirection({ evidenceNode: evidence });
expect(result.direction).toBe("cannot_determine");
});
it("returns cannot_determine when condition is null", () => {
const result = assessEvidenceDirection({ condition: null, evidenceNode: makeNode("t-1", "test") });
expect(result.direction).toBe("cannot_determine");
});
it("returns cannot_determine when evidenceNode is missing", () => {
const result = assessEvidenceDirection({ condition: { text: "demand" } });
expect(result.direction).toBe("cannot_determine");
});
it("returns cannot_determine when evidenceNode is null", () => {
const result = assessEvidenceDirection({
condition: { text: "demand" },
evidenceNode: null,
});
expect(result.direction).toBe("cannot_determine");
});
it("returns cannot_determine when condition has empty text", () => {
const result = assessEvidenceDirection({
condition: { text: "" },
evidenceNode: makeNode("t-1", "test"),
});
expect(result.direction).toBe("cannot_determine");
});
it("returns cannot_determine when evidenceNode has empty description", () => {
const result = assessEvidenceDirection({
condition: { text: DECISION_CONDITIONS[0] },
evidenceNode: makeNode("t-1", ""),
});
expect(result.direction).toBe("cannot_determine");
});
it("returns cannot_determine when neither input is provided", () => {
const result = assessEvidenceDirection({});
expect(result.direction).toBe("cannot_determine");
});
});
/* ── Deterministic output ───────────────────────────────── */
describe("assessEvidenceDirection — determinism", () => {
it("returns identical results for identical inputs across multiple calls", () => {
const evidence = makeNode("det-1", "European analytics market growing 15% annually");
const input = {
condition: { text: DECISION_CONDITIONS[0] },
evidenceNode: evidence,
};
const r1 = assessEvidenceDirection(input);
const r2 = assessEvidenceDirection(input);
expect(r1).toEqual(r2);
});
it("deterministic across 5 calls with long-investigation data", () => {
const fixture = buildScenarioFixture("long", 2);
const obs3Node = makeNode("obs-3", "Our platform does not support EU data residency requirements");
const input = {
condition: { text: DECISION_CONDITIONS[1] },
evidenceNode: obs3Node,
};
const results = [1, 2, 3, 4, 5].map(() => assessEvidenceDirection(input));
for (let i = 1; i < results.length; i++) {
expect(results[i]).toEqual(results[0]);
}
});
});
/* ── Input immutability ─────────────────────────────────── */
describe("assessEvidenceDirection — input immutability", () => {
it("does not mutate the condition object", () => {
const cond = { text: DECISION_CONDITIONS[0] };
const originalText = cond.text;
const evidence = makeNode("imm-1", "European market valued at €8B");
assessEvidenceDirection({ condition: cond, evidenceNode: evidence });
expect(cond.text).toBe(originalText);
});
it("does not mutate the evidenceNode object", () => {
const node = makeNode("imm-2", "test description");
const originalDesc = node.description;
const condition = { text: DECISION_CONDITIONS[3] };
assessEvidenceDirection({ condition, evidenceNode: node });
expect(node.description).toBe(originalDesc);
});
});
/* ── Four long-investigation examples ───────────────────── */
describe("assessEvidenceDirection — long investigation examples", () => {
it("demand example: supports (obs-2)", () => {
const fixture = buildScenarioFixture("long", 1);
const graphNodes = fixture.situationGraph.nodes;
const obs2Node = graphNodes.find((n) => n.id === "obs-2");
expect(obs2Node).toBeDefined();
const result = assessEvidenceDirection({
condition: { text: DECISION_CONDITIONS[0] },
evidenceNode: obs2Node,
});
expect(result.direction).toBe("supports");
});
it("compliance example: contradicts (obs-3)", () => {
const fixture = buildScenarioFixture("long", 2);
const graphNodes = fixture.situationGraph.nodes;
const obs3Node = graphNodes.find((n) => n.id === "obs-3");
expect(obs3Node).toBeDefined();
const result = assessEvidenceDirection({
condition: { text: DECISION_CONDITIONS[1] },
evidenceNode: obs3Node,
});
expect(result.direction).toBe("contradicts");
});
it("value versus cost example: informs (obs-4)", () => {
const fixture = buildScenarioFixture("long", 3);
const graphNodes = fixture.situationGraph.nodes;
const obs4Node = graphNodes.find((n) => n.id === "obs-4");
expect(obs4Node).toBeDefined();
const result = assessEvidenceDirection({
condition: { text: DECISION_CONDITIONS[2] },
evidenceNode: obs4Node,
});
expect(result.direction).toBe("informs");
});
it("differentiation example: supports (obs-5)", () => {
const fixture = buildScenarioFixture("long", 4);
const graphNodes = fixture.situationGraph.nodes;
const obs5Node = graphNodes.find((n) => n.id === "obs-5");
expect(obs5Node).toBeDefined();
const result = assessEvidenceDirection({
condition: { text: DECISION_CONDITIONS[3] },
evidenceNode: obs5Node,
});
expect(result.direction).toBe("supports");
});
});
/* ── Unrelated evidence ─────────────────────────────────── */
describe("assessEvidenceDirection — unrelated evidence", () => {
it("returns informs when evidence is about a different concept area", () => {
const evidence = makeNode("unrel-1", "Our team has 15 engineers in London");
const result = assessEvidenceDirection({
condition: { text: DECISION_CONDITIONS[0] },
evidenceNode: evidence,
});
expect(result.direction).toBe("informs");
});
it("returns informs when observation contains no category-matching keywords", () => {
const evidence = makeNode("unrel-2", "The office lease expires in December 2026");
const result = assessEvidenceDirection({
condition: { text: DECISION_CONDITIONS[1] },
evidenceNode: evidence,
});
expect(result.direction).toBe("informs");
});
});