experiment: test decision-relative coherence
This commit is contained in:
@@ -0,0 +1,582 @@
|
||||
/**
|
||||
* Experiment 51 — Is Coherence Relative to the Decision, Rather Than the Graph Shape?
|
||||
*
|
||||
* Passive diagnostic. Tests whether the existing passive decision-relevance classifier
|
||||
* (assessQuestionRelevanceToDecision) distinguishes:
|
||||
* Set A — several unknowns that all genuinely belong to one clear decision;
|
||||
* from
|
||||
* Set B — several unknowns present in the same investigation but not all relevant to it.
|
||||
*
|
||||
* The classifier was trained on European market entry scenarios (Exp 21). This experiment tests:
|
||||
* (1) within its vocabulary, can it distinguish coherent from scattered unknowns?
|
||||
* (2) outside its vocabulary (different domain), does it still produce varied results?
|
||||
* (3) when paraphrased, do the same underlying questions get different classifications?
|
||||
*
|
||||
* No production code changes. No active engine integration. Pure test-level evaluation.
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { assessQuestionRelevanceToDecision } from "@/lib/graph/question-decision-relevance.js";
|
||||
|
||||
/* ── Helpers ─────────────────────────────────────────────────── */
|
||||
|
||||
function makeUnknown(id, label) {
|
||||
return {
|
||||
id,
|
||||
label: label || `Unknown ${id}`,
|
||||
description: label || `Unknown ${id}`,
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "low",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
childIds: [],
|
||||
};
|
||||
}
|
||||
|
||||
function assessSet(set, decisionTarget) {
|
||||
return set.map((u) => ({
|
||||
id: u.id,
|
||||
label: u.label,
|
||||
...assessQuestionRelevanceToDecision({ decisionTarget, unknown: u }),
|
||||
}));
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Domain 1 — European Market Entry
|
||||
*
|
||||
* Decision target uses "enter" (a keyword the classifier recognises
|
||||
* in hasActionKeyword) and the question phrasing matches patterns from Exp 21.
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
const DOMAIN_1_DECISION =
|
||||
"Should we enter the European market with our SaaS analytics platform?";
|
||||
|
||||
/* ── Set A — Coherent Breadth (4 unknowns, all genuinely relevant) ─── */
|
||||
|
||||
const DOMAIN_1_COHERENT = [
|
||||
{ id: "demand", label: "Whether to enter the European market for analytics tools" },
|
||||
{
|
||||
id: "compliance",
|
||||
label: "Whether our product is suitable for European compliance requirements",
|
||||
},
|
||||
{
|
||||
id: "cost-benefit",
|
||||
label: "Whether the cost of achieving compliance is justified by the potential market size",
|
||||
},
|
||||
{
|
||||
id: "differentiation",
|
||||
label: "Whether we have competitive differentiation against existing European players",
|
||||
},
|
||||
];
|
||||
|
||||
/* ── Set B — Scattered Breadth (4 unknowns, mixed relevance) ─── */
|
||||
|
||||
const DOMAIN_1_SCATTERED = [
|
||||
{
|
||||
id: "scat-demand",
|
||||
label: "Whether we should enter the European market for analytics tools",
|
||||
},
|
||||
{
|
||||
id: "scat-staff-conflict",
|
||||
label: "Can two senior staff members resolve their ongoing disagreement?",
|
||||
},
|
||||
{
|
||||
id: "scat-lease",
|
||||
label: "Should the head office lease be renewed at the current rate next year?",
|
||||
},
|
||||
{
|
||||
id: "scat-pricing",
|
||||
label: "Does an existing unrelated product's pricing align with market willingness to pay?",
|
||||
},
|
||||
];
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Domain 2 — Community Event (different domain, different vocabulary)
|
||||
*
|
||||
* Tests whether the classifier generalises outside its training domain.
|
||||
* The decision target uses "organise" (not in hasActionKeyword),
|
||||
* and none of the questions match the Exp-21-specific patterns.
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
const DOMAIN_2_DECISION =
|
||||
"Should we organise the community event outdoors this September?";
|
||||
|
||||
const DOMAIN_2_COHERENT = [
|
||||
{ id: "evt-weather", label: "Whether there is sufficient weather risk for an outdoor event in September" },
|
||||
{
|
||||
id: "evt-insurance",
|
||||
label: "What insurance requirements apply for hosting the event outdoors",
|
||||
},
|
||||
{
|
||||
id: "evt-capacity",
|
||||
label: "Whether the outdoor venue can accommodate expected attendance",
|
||||
},
|
||||
{
|
||||
id: "evt-accessibility",
|
||||
label: "Whether the outdoor venue meets accessibility requirements for all attendees",
|
||||
},
|
||||
];
|
||||
|
||||
const DOMAIN_2_SCATTERED = [
|
||||
{ id: "scat-evt-weather", label: "Whether there is sufficient weather risk for an outdoor event in September" },
|
||||
{
|
||||
id: "scat-board-chairs",
|
||||
label: "Should the board replace its meeting room chairs next month?",
|
||||
},
|
||||
{
|
||||
id: "scat-volunteer",
|
||||
label: "Whether available volunteers can staff the registration desk on event day",
|
||||
},
|
||||
{
|
||||
id: "scat-local-park",
|
||||
label: "What local parks offer covered spaces in case of rain?",
|
||||
},
|
||||
];
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Domain 1 — Coherent set evaluation
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
describe("Experiment 51 — Domain 1 coherent set", () => {
|
||||
let results;
|
||||
|
||||
beforeAll(() => {
|
||||
const nodes = DOMAIN_1_COHERENT.map((u) => makeUnknown(u.id, u.label));
|
||||
results = assessSet(nodes, DOMAIN_1_DECISION);
|
||||
});
|
||||
|
||||
it("coherent set contains exactly four unknowns", () => {
|
||||
expect(results.length).toBe(4);
|
||||
});
|
||||
|
||||
it("demand unknown is classified as could_change_decision (matches DECISION_REVERSAL_PATTERNS)", () => {
|
||||
const r = results.find((r) => r.id === "demand");
|
||||
expect(r.relevance).toBe("could_change_decision");
|
||||
expect(typeof r.reason).toBe("string");
|
||||
expect(r.reason.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("compliance unknown is classified as supports_decision (matches PRECONDITION_PATTERNS)", () => {
|
||||
const r = results.find((r) => r.id === "compliance");
|
||||
expect(r.relevance).toBe("supports_decision");
|
||||
});
|
||||
|
||||
it("cost-benefit unknown is classified as supports_decision (matches FEASIBILITY_PATTERNS)", () => {
|
||||
const r = results.find((r) => r.id === "cost-benefit");
|
||||
expect(r.relevance).toBe("supports_decision");
|
||||
});
|
||||
|
||||
it("differentiation unknown is classified as supports_decision (matches SUPPORTING_CONTEXT_PATTERNS)", () => {
|
||||
const r = results.find((r) => r.id === "differentiation");
|
||||
expect(r.relevance).toBe("supports_decision");
|
||||
});
|
||||
|
||||
it("every unknown in the coherent set receives a meaningful classification (not cannot_determine)", () => {
|
||||
for (const r of results) {
|
||||
expect(r.relevance).not.toBe("cannot_determine");
|
||||
}
|
||||
});
|
||||
|
||||
it("every unknown includes a non-empty reason", () => {
|
||||
for (const r of results) {
|
||||
expect(typeof r.reason).toBe("string");
|
||||
expect(r.reason.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Domain 1 — Scattered set evaluation
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
describe("Experiment 51 — Domain 1 scattered set", () => {
|
||||
let results;
|
||||
|
||||
beforeAll(() => {
|
||||
const nodes = DOMAIN_1_SCATTERED.map((u) => makeUnknown(u.id, u.label));
|
||||
results = assessSet(nodes, DOMAIN_1_DECISION);
|
||||
});
|
||||
|
||||
it("scattered set contains exactly four unknowns", () => {
|
||||
expect(results.length).toBe(4);
|
||||
});
|
||||
|
||||
it("demand-like unknown (scat-demand) is classified as could_change_decision", () => {
|
||||
const r = results.find((r) => r.id === "scat-demand");
|
||||
expect(r.relevance).toBe("could_change_decision");
|
||||
});
|
||||
|
||||
it("staff conflict unknown produces cannot_determine or unlikely_to_change_decision (no pattern match)", () => {
|
||||
const r = results.find((r) => r.id === "scat-staff-conflict");
|
||||
expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
|
||||
});
|
||||
|
||||
it("lease unknown produces cannot_determine or unlikely_to_change_decision (no pattern match)", () => {
|
||||
const r = results.find((r) => r.id === "scat-lease");
|
||||
expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
|
||||
});
|
||||
|
||||
it("unrelated product pricing unknown produces cannot_determine or unlikely_to_change_decision (no pattern match)", () => {
|
||||
const r = results.find((r) => r.id === "scat-pricing");
|
||||
expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
|
||||
});
|
||||
|
||||
it("scattered set contains some classified as cannot_determine or unlikely_to_change_decision", () => {
|
||||
const nonRelevant = results.filter(
|
||||
(r) => r.relevance === "cannot_determine" || r.relevance === "unlikely_to_change_decision"
|
||||
).length;
|
||||
expect(nonRelevant).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
});
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Domain 1 — Direct comparison of coherent vs scattered sets
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
describe("Experiment 51 — Domain 1 coherent vs scattered comparison", () => {
|
||||
let coherentResults, scatteredResults;
|
||||
|
||||
beforeAll(() => {
|
||||
const coherentNodes = DOMAIN_1_COHERENT.map((u) => makeUnknown(u.id, u.label));
|
||||
const scatteredNodes = DOMAIN_1_SCATTERED.map((u) => makeUnknown(u.id, u.label));
|
||||
coherentResults = assessSet(coherentNodes, DOMAIN_1_DECISION);
|
||||
scatteredResults = assessSet(scatteredNodes, DOMAIN_1_DECISION);
|
||||
});
|
||||
|
||||
it("both sets contain the same number of unknowns", () => {
|
||||
expect(coherentResults.length).toBe(scatteredResults.length);
|
||||
});
|
||||
|
||||
it("coherent set produces all relevant classifications (could_change or supports)", () => {
|
||||
const coherentRelevant = coherentResults.filter(
|
||||
(r) => r.relevance === "could_change_decision" || r.relevance === "supports_decision"
|
||||
).length;
|
||||
expect(coherentRelevant).toBe(4); // all four are relevant to market entry
|
||||
});
|
||||
|
||||
it("scattered set contains at least two items classified as irrelevant or cannot_determine", () => {
|
||||
const scatteredNotRelevant = scatteredResults.filter(
|
||||
(r) => r.relevance === "cannot_determine" || r.relevance === "unlikely_to_change_decision"
|
||||
).length;
|
||||
expect(scatteredNotRelevant).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it("scattered demand-like question matches coherent demand-like classification (same pattern)", () => {
|
||||
const coherentDemand = coherentResults.find((r) => r.id === "demand");
|
||||
const scatteredDemandLike = scatteredResults.find((r) => r.id === "scat-demand");
|
||||
expect(scatteredDemandLike.relevance).toBe(coherentDemand.relevance);
|
||||
});
|
||||
|
||||
it("deterministic output for coherent set", () => {
|
||||
const nodes = DOMAIN_1_COHERENT.map((u) => makeUnknown(u.id, u.label));
|
||||
const r1 = assessSet(nodes, DOMAIN_1_DECISION);
|
||||
const r2 = assessSet([...nodes], DOMAIN_1_DECISION);
|
||||
expect(r1).toEqual(r2);
|
||||
});
|
||||
|
||||
it("deterministic output for scattered set", () => {
|
||||
const nodes = DOMAIN_1_SCATTERED.map((u) => makeUnknown(u.id, u.label));
|
||||
const r1 = assessSet(nodes, DOMAIN_1_DECISION);
|
||||
const r2 = assessSet([...nodes], DOMAIN_1_DECISION);
|
||||
expect(r1).toEqual(r2);
|
||||
});
|
||||
|
||||
it("inputs are not mutated", () => {
|
||||
const node = makeUnknown("mut-test", "Original label text");
|
||||
const originalLabel = node.label;
|
||||
const originalDescription = node.description;
|
||||
assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_1_DECISION, unknown: node });
|
||||
expect(node.label).toBe(originalLabel);
|
||||
expect(node.description).toBe(originalDescription);
|
||||
});
|
||||
|
||||
it("coherent set produces more than one distinct relevance category", () => {
|
||||
const categories = new Set(coherentResults.map((r) => r.relevance));
|
||||
expect(categories.size).toBeGreaterThan(1);
|
||||
});
|
||||
});
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Domain 2 — Coherent set evaluation (community event)
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
describe("Experiment 51 — Domain 2 coherent set", () => {
|
||||
let results;
|
||||
|
||||
beforeAll(() => {
|
||||
const nodes = DOMAIN_2_COHERENT.map((u) => makeUnknown(u.id, u.label));
|
||||
results = assessSet(nodes, DOMAIN_2_DECISION);
|
||||
});
|
||||
|
||||
it("coherent set contains exactly four unknowns", () => {
|
||||
expect(results.length).toBe(4);
|
||||
});
|
||||
|
||||
it("weather unknown is classified as cannot_determine (cannot generalise to non-market domain)", () => {
|
||||
const r = results.find((r) => r.id === "evt-weather");
|
||||
// The DECISION_REVERSAL_PATTERNS only match demand/market/need keywords — not "weather risk"
|
||||
// This demonstrates the classifier's language sensitivity
|
||||
expect(r.relevance).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("insurance unknown is classified as cannot_determine (no precondition/feasibility pattern match)", () => {
|
||||
const r = results.find((r) => r.id === "evt-insurance");
|
||||
expect(r.relevance).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("capacity unknown is classified as cannot_determine (generic capacity language not in any pattern)", () => {
|
||||
const r = results.find((r) => r.id === "evt-capacity");
|
||||
expect(r.relevance).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("accessibility unknown produces cannot_determine or unlikely_to_change_decision", () => {
|
||||
const r = results.find((r) => r.id === "evt-accessibility");
|
||||
expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
|
||||
});
|
||||
|
||||
it("coherent set is mixed — not all classified as relevant (classifier cannot generalise)", () => {
|
||||
const categories = new Set(results.map((r) => r.relevance));
|
||||
expect(categories.has("cannot_determine")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Domain 2 — Scattered set evaluation (community event)
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
describe("Experiment 51 — Domain 2 scattered set", () => {
|
||||
let results;
|
||||
|
||||
beforeAll(() => {
|
||||
const nodes = DOMAIN_2_SCATTERED.map((u) => makeUnknown(u.id, u.label));
|
||||
results = assessSet(nodes, DOMAIN_2_DECISION);
|
||||
});
|
||||
|
||||
it("scattered set contains exactly four unknowns", () => {
|
||||
expect(results.length).toBe(4);
|
||||
});
|
||||
|
||||
it("board chairs unknown is classified as cannot_determine or unlikely_to_change_decision", () => {
|
||||
const r = results.find((r) => r.id === "scat-board-chairs");
|
||||
expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
|
||||
});
|
||||
|
||||
it("weather question (appears in both coherent and scattered sets) produces cannot_determine outside training domain", () => {
|
||||
const scatteredWeather = results.find((r) => r.id === "scat-evt-weather");
|
||||
// Same phrasing as Domain 2 coherent — both produce cannot_determine
|
||||
expect(scatteredWeather.relevance).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("volunteer unknown is classified as cannot_determine or unlikely_to_change_decision", () => {
|
||||
const r = results.find((r) => r.id === "scat-volunteer");
|
||||
expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
|
||||
});
|
||||
|
||||
it("local park unknown is classified as cannot_determine or unlikely_to_change_decision", () => {
|
||||
const r = results.find((r) => r.id === "scat-local-park");
|
||||
expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
|
||||
});
|
||||
});
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Domain 2 — Coherent vs scattered comparison (community event)
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
describe("Experiment 51 — Domain 2 coherent vs scattered comparison", () => {
|
||||
let coherentResults, scatteredResults;
|
||||
|
||||
beforeAll(() => {
|
||||
const coherentNodes = DOMAIN_2_COHERENT.map((u) => makeUnknown(u.id, u.label));
|
||||
const scatteredNodes = DOMAIN_2_SCATTERED.map((u) => makeUnknown(u.id, u.label));
|
||||
coherentResults = assessSet(coherentNodes, DOMAIN_2_DECISION);
|
||||
scatteredResults = assessSet(scatteredNodes, DOMAIN_2_DECISION);
|
||||
});
|
||||
|
||||
it("both sets contain the same number of unknowns", () => {
|
||||
expect(coherentResults.length).toBe(scatteredResults.length);
|
||||
});
|
||||
|
||||
it("domain-2 coherent set does NOT produce all relevant classifications (classifier cannot generalise)", () => {
|
||||
const coherentRelevant = coherentResults.filter(
|
||||
(r) => r.relevance === "could_change_decision" || r.relevance === "supports_decision"
|
||||
).length;
|
||||
// Only the weather question matches a pattern — rest are cannot_determine
|
||||
expect(coherentRelevant).toBeLessThan(4);
|
||||
});
|
||||
|
||||
it("both domains' coherent sets produce fewer than all-relevant classifications (Domain 2 specifically)", () => {
|
||||
const categories = new Set(coherentResults.map((r) => r.relevance));
|
||||
expect(categories.has("cannot_determine")).toBe(true);
|
||||
});
|
||||
|
||||
it("deterministic output for both sets", () => {
|
||||
const cn = DOMAIN_2_COHERENT.map((u) => makeUnknown(u.id, u.label));
|
||||
const sn = DOMAIN_2_SCATTERED.map((u) => makeUnknown(u.id, u.label));
|
||||
expect(assessSet([...cn], DOMAIN_2_DECISION)).toEqual(assessSet(cn, DOMAIN_2_DECISION));
|
||||
expect(assessSet([...sn], DOMAIN_2_DECISION)).toEqual(assessSet(sn, DOMAIN_2_DECISION));
|
||||
});
|
||||
});
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Language Robustness — Paraphrase evaluation
|
||||
*
|
||||
* One coherent paraphrase and one unrelated paraphrase.
|
||||
* These avoid the most obvious wording from the Exp-21 originals
|
||||
* to test whether the classifier understands relevance or just
|
||||
* recognises familiar keywords.
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
describe("Experiment 51 — Language robustness (paraphrases)", () => {
|
||||
let coherentOriginal;
|
||||
let coherentParaphrase;
|
||||
let unrelatedOriginal;
|
||||
let unrelatedParaphrase;
|
||||
let coherentOriginalResult;
|
||||
let coherentParaphraseResult;
|
||||
let unrelatedOriginalResult;
|
||||
let unrelatedParaphraseResult;
|
||||
|
||||
beforeAll(() => {
|
||||
// Coherent original uses phrasing the classifier recognises (Exp 21 pattern)
|
||||
coherentOriginal = makeUnknown("coh-orig", "Whether to enter the European market for analytics tools");
|
||||
// Coherent paraphrase: avoids "enter", "European market" — uses plain English
|
||||
coherentParaphrase = makeUnknown("coh-paraphrased", "Would enough people there actually want what we offer?");
|
||||
|
||||
// Unrelated original matches Exp 21's unlikely pattern (benchmark keyword)
|
||||
unrelatedOriginal = makeUnknown("unrel-orig", "What benchmarks do other SaaS companies use for market sizing");
|
||||
// Unrelated paraphrase: avoids "benchmark" — uses different phrasing
|
||||
unrelatedParaphrase = makeUnknown("unrel-paraphrased", "Which analytics firms set the industry standard?");
|
||||
|
||||
coherentOriginalResult = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_1_DECISION, unknown: coherentOriginal });
|
||||
coherentParaphraseResult = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_1_DECISION, unknown: coherentParaphrase });
|
||||
unrelatedOriginalResult = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_1_DECISION, unknown: unrelatedOriginal });
|
||||
unrelatedParaphraseResult = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_1_DECISION, unknown: unrelatedParaphrase });
|
||||
});
|
||||
|
||||
it("coherent paraphrase produces cannot_determine (classifier does not recognise the phrasing)", () => {
|
||||
expect(coherentParaphraseResult.relevance).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("unrelated paraphrase produces cannot_determine or unlikely_to_change_decision", () => {
|
||||
expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(unrelatedParaphraseResult.relevance);
|
||||
});
|
||||
|
||||
it("coherent original and coherent paraphrase produce different classifications (language sensitivity)", () => {
|
||||
expect(coherentOriginalResult.relevance).not.toBe(coherentParaphraseResult.relevance);
|
||||
});
|
||||
|
||||
it("unrelated paraphrase is NOT classified as relevant (no regression from paraphrase)", () => {
|
||||
expect(unrelatedParaphraseResult.relevance).not.toBe("could_change_decision");
|
||||
expect(unrelatedParaphraseResult.relevance).not.toBe("supports_decision");
|
||||
});
|
||||
|
||||
it("both coherent and unrelated produce non-empty reasons (or cannot_determine with reason)", () => {
|
||||
if (coherentOriginalResult.relevance !== "cannot_determine") {
|
||||
expect(typeof coherentOriginalResult.reason).toBe("string");
|
||||
expect(coherentOriginalResult.reason.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Production classifier unchanged — verification against Exp-21 baseline
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
describe("Experiment 51 — Classifier behaviour consistency check", () => {
|
||||
it("classifier returns same categories as Experiment 21 for known patterns", () => {
|
||||
const goNoGo = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: "Should we enter the European market?",
|
||||
unknown: makeUnknown("v1", "Whether to proceed with European market entry"),
|
||||
});
|
||||
expect(goNoGo.relevance).toBe("could_change_decision");
|
||||
|
||||
const compliance = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: "Should we enter the European market?",
|
||||
unknown: makeUnknown("v2", "Whether our product is suitable for European compliance requirements"),
|
||||
});
|
||||
expect(compliance.relevance).toBe("supports_decision");
|
||||
|
||||
const benchmark = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: "Should we enter the European market?",
|
||||
unknown: makeUnknown("v3", "What benchmarks do other SaaS companies use for market sizing"),
|
||||
});
|
||||
expect(benchmark.relevance).toBe("unlikely_to_change_decision");
|
||||
|
||||
const costBenefit = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: "Should we enter the European market?",
|
||||
unknown: makeUnknown("v4", "Whether the cost of achieving compliance is justified by the market size"),
|
||||
});
|
||||
expect(costBenefit.relevance).toBe("supports_decision");
|
||||
|
||||
const compDiff = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: "Should we enter the European market?",
|
||||
unknown: makeUnknown("v5", "Whether we have competitive differentiation against existing European players"),
|
||||
});
|
||||
expect(compDiff.relevance).toBe("supports_decision");
|
||||
});
|
||||
|
||||
it("classifier does NOT use the decision target for semantic relevance (tests two decisions with same unknown)", () => {
|
||||
// dt1 contains "enter" (an action keyword that enables Rule 1).
|
||||
// dt2 does not contain any action keywords.
|
||||
// Both describe completely unrelated topics (market entry vs. budget).
|
||||
const dt1 = "Should we enter the European market?";
|
||||
const dt2 = "What is the budget for next fiscal year";
|
||||
|
||||
const result1 = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: dt1,
|
||||
unknown: makeUnknown("same-q", "Whether to proceed with European market entry"),
|
||||
});
|
||||
const result2 = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: dt2,
|
||||
unknown: makeUnknown("same-q", "Whether to proceed with European market entry"),
|
||||
});
|
||||
|
||||
// The classifier does NOT compare the unknown's meaning against the decision target.
|
||||
// It only checks whether the decision text contains one of five action keywords
|
||||
// (enter/launch/build/stop/abandon) for gating Rule 1.
|
||||
// dt1 has "enter" → Rule 1 succeeds → could_change_decision
|
||||
// dt2 has no action keyword → Rule 1 skips → cannot_determine
|
||||
// This proves: the decision target never provides semantic context for matching.
|
||||
expect(result1.relevance).toBe("could_change_decision");
|
||||
expect(result2.relevance).toBe("cannot_determine");
|
||||
});
|
||||
|
||||
it("classifier produces more than one category when presented with varied inputs against same decision", () => {
|
||||
const dt = "Should we enter the European market?";
|
||||
const unknowns = [
|
||||
makeUnknown("var1", "Whether to proceed with European market entry"),
|
||||
makeUnknown("var2", "What benchmarks do other SaaS companies use for market sizing"),
|
||||
makeUnknown("var3", "Whether our product is suitable for European compliance requirements"),
|
||||
];
|
||||
const results = unknowns.map((u) => assessQuestionRelevanceToDecision({ decisionTarget: dt, unknown: u }));
|
||||
const categories = new Set(results.map((r) => r.relevance));
|
||||
expect(categories.size).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it("missing inputs produce cannot_determine (not collapse to a single category)", () => {
|
||||
const dt = "Should we enter the European market?";
|
||||
|
||||
const noDecision = assessQuestionRelevanceToDecision({
|
||||
unknown: makeUnknown("no-dt", "Whether to proceed"),
|
||||
});
|
||||
expect(noDecision.relevance).toBe("cannot_determine");
|
||||
|
||||
const emptyDecision = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: "",
|
||||
unknown: makeUnknown("empty-dt", "Whether to proceed"),
|
||||
});
|
||||
expect(emptyDecision.relevance).toBe("cannot_determine");
|
||||
|
||||
const noUnknown = assessQuestionRelevanceToDecision({
|
||||
decisionTarget: dt,
|
||||
});
|
||||
expect(noUnknown.relevance).toBe("cannot_determine");
|
||||
|
||||
const noInput = assessQuestionRelevanceToDecision(null);
|
||||
expect(noInput.relevance).toBe("cannot_determine");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user