confidence-engine/tests/graph/decision-relative-coherence.test.js

/**
 * Experiment 51 — Is Coherence Relative to the Decision, Rather Than the Graph Shape?
 *
 * Passive diagnostic. Tests whether the existing passive decision-relevance classifier
 * (assessQuestionRelevanceToDecision) distinguishes:
 *   Set A — several unknowns that all genuinely belong to one clear decision;
 *   from
 *   Set B — several unknowns present in the same investigation but not all relevant to it.
 *
 * The classifier was trained on European market entry scenarios (Exp 21). This experiment tests:
 * (1) within its vocabulary, can it distinguish coherent from scattered unknowns?
 * (2) outside its vocabulary (different domain), does it still produce varied results?
 * (3) when paraphrased, do the same underlying questions get different classifications?
 *
 * No production code changes. No active engine integration. Pure test-level evaluation.
 */

import { describe, expect, it } from "vitest";
import { assessQuestionRelevanceToDecision } from "@/lib/graph/question-decision-relevance.js";

/* ── Helpers ─────────────────────────────────────────────────── */

function makeUnknown(id, label) {
  return {
    id,
    label: label || `Unknown ${id}`,
    description: label || `Unknown ${id}`,
    kind: "unknown",
    status: "unknown",
    confidence: "low",
    value: null,
    unit: null,
    evidenceIds: [],
    dependsOn: [],
    affects: [],
    childIds: [],
  };
}

function assessSet(set, decisionTarget) {
  return set.map((u) => ({
    id: u.id,
    label: u.label,
    ...assessQuestionRelevanceToDecision({ decisionTarget, unknown: u }),
  }));
}

/* ═══════════════════════════════════════════════════════════
 * Domain 1 — European Market Entry
 *
 * Decision target uses "enter" (a keyword the classifier recognises
 * in hasActionKeyword) and the question phrasing matches patterns from Exp 21.
 * ═══════════════════════════════════════════════════════════ */

const DOMAIN_1_DECISION =
  "Should we enter the European market with our SaaS analytics platform?";

/* ── Set A — Coherent Breadth (4 unknowns, all genuinely relevant) ─── */

const DOMAIN_1_COHERENT = [
  { id: "demand", label: "Whether to enter the European market for analytics tools" },
  {
    id: "compliance",
    label: "Whether our product is suitable for European compliance requirements",
  },
  {
    id: "cost-benefit",
    label: "Whether the cost of achieving compliance is justified by the potential market size",
  },
  {
    id: "differentiation",
    label: "Whether we have competitive differentiation against existing European players",
  },
];

/* ── Set B — Scattered Breadth (4 unknowns, mixed relevance) ─── */

const DOMAIN_1_SCATTERED = [
  {
    id: "scat-demand",
    label: "Whether we should enter the European market for analytics tools",
  },
  {
    id: "scat-staff-conflict",
    label: "Can two senior staff members resolve their ongoing disagreement?",
  },
  {
    id: "scat-lease",
    label: "Should the head office lease be renewed at the current rate next year?",
  },
  {
    id: "scat-pricing",
    label: "Does an existing unrelated product's pricing align with market willingness to pay?",
  },
];

/* ═══════════════════════════════════════════════════════════
 * Domain 2 — Community Event (different domain, different vocabulary)
 *
 * Tests whether the classifier generalises outside its training domain.
 * The decision target uses "organise" (not in hasActionKeyword),
 * and none of the questions match the Exp-21-specific patterns.
 * ═══════════════════════════════════════════════════════════ */

const DOMAIN_2_DECISION =
  "Should we organise the community event outdoors this September?";

const DOMAIN_2_COHERENT = [
  { id: "evt-weather", label: "Whether there is sufficient weather risk for an outdoor event in September" },
  {
    id: "evt-insurance",
    label: "What insurance requirements apply for hosting the event outdoors",
  },
  {
    id: "evt-capacity",
    label: "Whether the outdoor venue can accommodate expected attendance",
  },
  {
    id: "evt-accessibility",
    label: "Whether the outdoor venue meets accessibility requirements for all attendees",
  },
];

const DOMAIN_2_SCATTERED = [
  { id: "scat-evt-weather", label: "Whether there is sufficient weather risk for an outdoor event in September" },
  {
    id: "scat-board-chairs",
    label: "Should the board replace its meeting room chairs next month?",
  },
  {
    id: "scat-volunteer",
    label: "Whether available volunteers can staff the registration desk on event day",
  },
  {
    id: "scat-local-park",
    label: "What local parks offer covered spaces in case of rain?",
  },
];

/* ═══════════════════════════════════════════════════════════
 * Domain 1 — Coherent set evaluation
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 51 — Domain 1 coherent set", () => {
  let results;

  beforeAll(() => {
    const nodes = DOMAIN_1_COHERENT.map((u) => makeUnknown(u.id, u.label));
    results = assessSet(nodes, DOMAIN_1_DECISION);
  });

  it("coherent set contains exactly four unknowns", () => {
    expect(results.length).toBe(4);
  });

  it("demand unknown is classified as could_change_decision (matches DECISION_REVERSAL_PATTERNS)", () => {
    const r = results.find((r) => r.id === "demand");
    expect(r.relevance).toBe("could_change_decision");
    expect(typeof r.reason).toBe("string");
    expect(r.reason.length).toBeGreaterThan(0);
  });

  it("compliance unknown is classified as supports_decision (matches PRECONDITION_PATTERNS)", () => {
    const r = results.find((r) => r.id === "compliance");
    expect(r.relevance).toBe("supports_decision");
  });

  it("cost-benefit unknown is classified as supports_decision (matches FEASIBILITY_PATTERNS)", () => {
    const r = results.find((r) => r.id === "cost-benefit");
    expect(r.relevance).toBe("supports_decision");
  });

  it("differentiation unknown is classified as supports_decision (matches SUPPORTING_CONTEXT_PATTERNS)", () => {
    const r = results.find((r) => r.id === "differentiation");
    expect(r.relevance).toBe("supports_decision");
  });

  it("every unknown in the coherent set receives a meaningful classification (not cannot_determine)", () => {
    for (const r of results) {
      expect(r.relevance).not.toBe("cannot_determine");
    }
  });

  it("every unknown includes a non-empty reason", () => {
    for (const r of results) {
      expect(typeof r.reason).toBe("string");
      expect(r.reason.length).toBeGreaterThan(0);
    }
  });
});

/* ═══════════════════════════════════════════════════════════
 * Domain 1 — Scattered set evaluation
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 51 — Domain 1 scattered set", () => {
  let results;

  beforeAll(() => {
    const nodes = DOMAIN_1_SCATTERED.map((u) => makeUnknown(u.id, u.label));
    results = assessSet(nodes, DOMAIN_1_DECISION);
  });

  it("scattered set contains exactly four unknowns", () => {
    expect(results.length).toBe(4);
  });

  it("demand-like unknown (scat-demand) is classified as could_change_decision", () => {
    const r = results.find((r) => r.id === "scat-demand");
    expect(r.relevance).toBe("could_change_decision");
  });

  it("staff conflict unknown produces cannot_determine or unlikely_to_change_decision (no pattern match)", () => {
    const r = results.find((r) => r.id === "scat-staff-conflict");
    expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
  });

  it("lease unknown produces cannot_determine or unlikely_to_change_decision (no pattern match)", () => {
    const r = results.find((r) => r.id === "scat-lease");
    expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
  });

  it("unrelated product pricing unknown produces cannot_determine or unlikely_to_change_decision (no pattern match)", () => {
    const r = results.find((r) => r.id === "scat-pricing");
    expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
  });

  it("scattered set contains some classified as cannot_determine or unlikely_to_change_decision", () => {
    const nonRelevant = results.filter(
      (r) => r.relevance === "cannot_determine" || r.relevance === "unlikely_to_change_decision"
    ).length;
    expect(nonRelevant).toBeGreaterThanOrEqual(2);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Domain 1 — Direct comparison of coherent vs scattered sets
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 51 — Domain 1 coherent vs scattered comparison", () => {
  let coherentResults, scatteredResults;

  beforeAll(() => {
    const coherentNodes = DOMAIN_1_COHERENT.map((u) => makeUnknown(u.id, u.label));
    const scatteredNodes = DOMAIN_1_SCATTERED.map((u) => makeUnknown(u.id, u.label));
    coherentResults = assessSet(coherentNodes, DOMAIN_1_DECISION);
    scatteredResults = assessSet(scatteredNodes, DOMAIN_1_DECISION);
  });

  it("both sets contain the same number of unknowns", () => {
    expect(coherentResults.length).toBe(scatteredResults.length);
  });

  it("coherent set produces all relevant classifications (could_change or supports)", () => {
    const coherentRelevant = coherentResults.filter(
      (r) => r.relevance === "could_change_decision" || r.relevance === "supports_decision"
    ).length;
    expect(coherentRelevant).toBe(4); // all four are relevant to market entry
  });

  it("scattered set contains at least two items classified as irrelevant or cannot_determine", () => {
    const scatteredNotRelevant = scatteredResults.filter(
      (r) => r.relevance === "cannot_determine" || r.relevance === "unlikely_to_change_decision"
    ).length;
    expect(scatteredNotRelevant).toBeGreaterThanOrEqual(2);
  });

  it("scattered demand-like question matches coherent demand-like classification (same pattern)", () => {
    const coherentDemand = coherentResults.find((r) => r.id === "demand");
    const scatteredDemandLike = scatteredResults.find((r) => r.id === "scat-demand");
    expect(scatteredDemandLike.relevance).toBe(coherentDemand.relevance);
  });

  it("deterministic output for coherent set", () => {
    const nodes = DOMAIN_1_COHERENT.map((u) => makeUnknown(u.id, u.label));
    const r1 = assessSet(nodes, DOMAIN_1_DECISION);
    const r2 = assessSet([...nodes], DOMAIN_1_DECISION);
    expect(r1).toEqual(r2);
  });

  it("deterministic output for scattered set", () => {
    const nodes = DOMAIN_1_SCATTERED.map((u) => makeUnknown(u.id, u.label));
    const r1 = assessSet(nodes, DOMAIN_1_DECISION);
    const r2 = assessSet([...nodes], DOMAIN_1_DECISION);
    expect(r1).toEqual(r2);
  });

  it("inputs are not mutated", () => {
    const node = makeUnknown("mut-test", "Original label text");
    const originalLabel = node.label;
    const originalDescription = node.description;
    assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_1_DECISION, unknown: node });
    expect(node.label).toBe(originalLabel);
    expect(node.description).toBe(originalDescription);
  });

  it("coherent set produces more than one distinct relevance category", () => {
    const categories = new Set(coherentResults.map((r) => r.relevance));
    expect(categories.size).toBeGreaterThan(1);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Domain 2 — Coherent set evaluation (community event)
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 51 — Domain 2 coherent set", () => {
  let results;

  beforeAll(() => {
    const nodes = DOMAIN_2_COHERENT.map((u) => makeUnknown(u.id, u.label));
    results = assessSet(nodes, DOMAIN_2_DECISION);
  });

  it("coherent set contains exactly four unknowns", () => {
    expect(results.length).toBe(4);
  });

  it("weather unknown is classified as cannot_determine (cannot generalise to non-market domain)", () => {
    const r = results.find((r) => r.id === "evt-weather");
    // The DECISION_REVERSAL_PATTERNS only match demand/market/need keywords — not "weather risk"
    // This demonstrates the classifier's language sensitivity
    expect(r.relevance).toBe("cannot_determine");
  });

  it("insurance unknown is classified as cannot_determine (no precondition/feasibility pattern match)", () => {
    const r = results.find((r) => r.id === "evt-insurance");
    expect(r.relevance).toBe("cannot_determine");
  });

  it("capacity unknown is classified as cannot_determine (generic capacity language not in any pattern)", () => {
    const r = results.find((r) => r.id === "evt-capacity");
    expect(r.relevance).toBe("cannot_determine");
  });

  it("accessibility unknown produces cannot_determine or unlikely_to_change_decision", () => {
    const r = results.find((r) => r.id === "evt-accessibility");
    expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
  });

  it("coherent set is mixed — not all classified as relevant (classifier cannot generalise)", () => {
    const categories = new Set(results.map((r) => r.relevance));
    expect(categories.has("cannot_determine")).toBe(true);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Domain 2 — Scattered set evaluation (community event)
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 51 — Domain 2 scattered set", () => {
  let results;

  beforeAll(() => {
    const nodes = DOMAIN_2_SCATTERED.map((u) => makeUnknown(u.id, u.label));
    results = assessSet(nodes, DOMAIN_2_DECISION);
  });

  it("scattered set contains exactly four unknowns", () => {
    expect(results.length).toBe(4);
  });

  it("board chairs unknown is classified as cannot_determine or unlikely_to_change_decision", () => {
    const r = results.find((r) => r.id === "scat-board-chairs");
    expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
  });

  it("weather question (appears in both coherent and scattered sets) produces cannot_determine outside training domain", () => {
    const scatteredWeather = results.find((r) => r.id === "scat-evt-weather");
    // Same phrasing as Domain 2 coherent — both produce cannot_determine
    expect(scatteredWeather.relevance).toBe("cannot_determine");
  });

  it("volunteer unknown is classified as cannot_determine or unlikely_to_change_decision", () => {
    const r = results.find((r) => r.id === "scat-volunteer");
    expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
  });

  it("local park unknown is classified as cannot_determine or unlikely_to_change_decision", () => {
    const r = results.find((r) => r.id === "scat-local-park");
    expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Domain 2 — Coherent vs scattered comparison (community event)
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 51 — Domain 2 coherent vs scattered comparison", () => {
  let coherentResults, scatteredResults;

  beforeAll(() => {
    const coherentNodes = DOMAIN_2_COHERENT.map((u) => makeUnknown(u.id, u.label));
    const scatteredNodes = DOMAIN_2_SCATTERED.map((u) => makeUnknown(u.id, u.label));
    coherentResults = assessSet(coherentNodes, DOMAIN_2_DECISION);
    scatteredResults = assessSet(scatteredNodes, DOMAIN_2_DECISION);
  });

  it("both sets contain the same number of unknowns", () => {
    expect(coherentResults.length).toBe(scatteredResults.length);
  });

  it("domain-2 coherent set does NOT produce all relevant classifications (classifier cannot generalise)", () => {
    const coherentRelevant = coherentResults.filter(
      (r) => r.relevance === "could_change_decision" || r.relevance === "supports_decision"
    ).length;
    // Only the weather question matches a pattern — rest are cannot_determine
    expect(coherentRelevant).toBeLessThan(4);
  });

  it("both domains' coherent sets produce fewer than all-relevant classifications (Domain 2 specifically)", () => {
    const categories = new Set(coherentResults.map((r) => r.relevance));
    expect(categories.has("cannot_determine")).toBe(true);
  });

  it("deterministic output for both sets", () => {
    const cn = DOMAIN_2_COHERENT.map((u) => makeUnknown(u.id, u.label));
    const sn = DOMAIN_2_SCATTERED.map((u) => makeUnknown(u.id, u.label));
    expect(assessSet([...cn], DOMAIN_2_DECISION)).toEqual(assessSet(cn, DOMAIN_2_DECISION));
    expect(assessSet([...sn], DOMAIN_2_DECISION)).toEqual(assessSet(sn, DOMAIN_2_DECISION));
  });
});

/* ═══════════════════════════════════════════════════════════
 * Language Robustness — Paraphrase evaluation
 *
 * One coherent paraphrase and one unrelated paraphrase.
 * These avoid the most obvious wording from the Exp-21 originals
 * to test whether the classifier understands relevance or just
 * recognises familiar keywords.
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 51 — Language robustness (paraphrases)", () => {
  let coherentOriginal;
  let coherentParaphrase;
  let unrelatedOriginal;
  let unrelatedParaphrase;
  let coherentOriginalResult;
  let coherentParaphraseResult;
  let unrelatedOriginalResult;
  let unrelatedParaphraseResult;

  beforeAll(() => {
    // Coherent original uses phrasing the classifier recognises (Exp 21 pattern)
    coherentOriginal = makeUnknown("coh-orig", "Whether to enter the European market for analytics tools");
    // Coherent paraphrase: avoids "enter", "European market" — uses plain English
    coherentParaphrase = makeUnknown("coh-paraphrased", "Would enough people there actually want what we offer?");

    // Unrelated original matches Exp 21's unlikely pattern (benchmark keyword)
    unrelatedOriginal = makeUnknown("unrel-orig", "What benchmarks do other SaaS companies use for market sizing");
    // Unrelated paraphrase: avoids "benchmark" — uses different phrasing
    unrelatedParaphrase = makeUnknown("unrel-paraphrased", "Which analytics firms set the industry standard?");

    coherentOriginalResult = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_1_DECISION, unknown: coherentOriginal });
    coherentParaphraseResult = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_1_DECISION, unknown: coherentParaphrase });
    unrelatedOriginalResult = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_1_DECISION, unknown: unrelatedOriginal });
    unrelatedParaphraseResult = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_1_DECISION, unknown: unrelatedParaphrase });
  });

  it("coherent paraphrase produces cannot_determine (classifier does not recognise the phrasing)", () => {
    expect(coherentParaphraseResult.relevance).toBe("cannot_determine");
  });

  it("unrelated paraphrase produces cannot_determine or unlikely_to_change_decision", () => {
    expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(unrelatedParaphraseResult.relevance);
  });

  it("coherent original and coherent paraphrase produce different classifications (language sensitivity)", () => {
    expect(coherentOriginalResult.relevance).not.toBe(coherentParaphraseResult.relevance);
  });

  it("unrelated paraphrase is NOT classified as relevant (no regression from paraphrase)", () => {
    expect(unrelatedParaphraseResult.relevance).not.toBe("could_change_decision");
    expect(unrelatedParaphraseResult.relevance).not.toBe("supports_decision");
  });

  it("both coherent and unrelated produce non-empty reasons (or cannot_determine with reason)", () => {
    if (coherentOriginalResult.relevance !== "cannot_determine") {
      expect(typeof coherentOriginalResult.reason).toBe("string");
      expect(coherentOriginalResult.reason.length).toBeGreaterThan(0);
    }
  });
});

/* ═══════════════════════════════════════════════════════════
 * Production classifier unchanged — verification against Exp-21 baseline
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 51 — Classifier behaviour consistency check", () => {
  it("classifier returns same categories as Experiment 21 for known patterns", () => {
    const goNoGo = assessQuestionRelevanceToDecision({
      decisionTarget: "Should we enter the European market?",
      unknown: makeUnknown("v1", "Whether to proceed with European market entry"),
    });
    expect(goNoGo.relevance).toBe("could_change_decision");

    const compliance = assessQuestionRelevanceToDecision({
      decisionTarget: "Should we enter the European market?",
      unknown: makeUnknown("v2", "Whether our product is suitable for European compliance requirements"),
    });
    expect(compliance.relevance).toBe("supports_decision");

    const benchmark = assessQuestionRelevanceToDecision({
      decisionTarget: "Should we enter the European market?",
      unknown: makeUnknown("v3", "What benchmarks do other SaaS companies use for market sizing"),
    });
    expect(benchmark.relevance).toBe("unlikely_to_change_decision");

    const costBenefit = assessQuestionRelevanceToDecision({
      decisionTarget: "Should we enter the European market?",
      unknown: makeUnknown("v4", "Whether the cost of achieving compliance is justified by the market size"),
    });
    expect(costBenefit.relevance).toBe("supports_decision");

    const compDiff = assessQuestionRelevanceToDecision({
      decisionTarget: "Should we enter the European market?",
      unknown: makeUnknown("v5", "Whether we have competitive differentiation against existing European players"),
    });
    expect(compDiff.relevance).toBe("supports_decision");
  });

  it("classifier does NOT use the decision target for semantic relevance (tests two decisions with same unknown)", () => {
    // dt1 contains "enter" (an action keyword that enables Rule 1).
    // dt2 does not contain any action keywords.
    // Both describe completely unrelated topics (market entry vs. budget).
    const dt1 = "Should we enter the European market?";
    const dt2 = "What is the budget for next fiscal year";

    const result1 = assessQuestionRelevanceToDecision({
      decisionTarget: dt1,
      unknown: makeUnknown("same-q", "Whether to proceed with European market entry"),
    });
    const result2 = assessQuestionRelevanceToDecision({
      decisionTarget: dt2,
      unknown: makeUnknown("same-q", "Whether to proceed with European market entry"),
    });

    // The classifier does NOT compare the unknown's meaning against the decision target.
    // It only checks whether the decision text contains one of five action keywords
    // (enter/launch/build/stop/abandon) for gating Rule 1.
    // dt1 has "enter" → Rule 1 succeeds → could_change_decision
    // dt2 has no action keyword → Rule 1 skips → cannot_determine
    // This proves: the decision target never provides semantic context for matching.
    expect(result1.relevance).toBe("could_change_decision");
    expect(result2.relevance).toBe("cannot_determine");
  });

  it("classifier produces more than one category when presented with varied inputs against same decision", () => {
    const dt = "Should we enter the European market?";
    const unknowns = [
      makeUnknown("var1", "Whether to proceed with European market entry"),
      makeUnknown("var2", "What benchmarks do other SaaS companies use for market sizing"),
      makeUnknown("var3", "Whether our product is suitable for European compliance requirements"),
    ];
    const results = unknowns.map((u) => assessQuestionRelevanceToDecision({ decisionTarget: dt, unknown: u }));
    const categories = new Set(results.map((r) => r.relevance));
    expect(categories.size).toBeGreaterThan(1);
  });

  it("missing inputs produce cannot_determine (not collapse to a single category)", () => {
    const dt = "Should we enter the European market?";

    const noDecision = assessQuestionRelevanceToDecision({
      unknown: makeUnknown("no-dt", "Whether to proceed"),
    });
    expect(noDecision.relevance).toBe("cannot_determine");

    const emptyDecision = assessQuestionRelevanceToDecision({
      decisionTarget: "",
      unknown: makeUnknown("empty-dt", "Whether to proceed"),
    });
    expect(emptyDecision.relevance).toBe("cannot_determine");

    const noUnknown = assessQuestionRelevanceToDecision({
      decisionTarget: dt,
    });
    expect(noUnknown.relevance).toBe("cannot_determine");

    const noInput = assessQuestionRelevanceToDecision(null);
    expect(noInput.relevance).toBe("cannot_determine");
  });
});