confidence-engine/tests/graph/decision-relevance-category-boundary.test.js

/**
 * Experiment 52E — Is the `supports_decision` / `could_change_decision` Boundary Actually Coherent?
 *
 * Passive contract-boundary experiment. Tests whether the existing distinction between
 * `could_change_decision` and `supports_decision` holds consistently across three explicit
 * contrast pairs (blocker vs supporting-evidence) in three distinct domains.
 *
 * Uses exactly the same category definitions and normalisation instruction as Experiment 52D.
 * Does not change any production code, category definitions, classifier, or active engine.
 */

import dotenv from "dotenv";
dotenv.config({ path: ".env.local" });

import { describe, it, expect, beforeAll } from "vitest";

/* ═══════════════════════════════════════════════════════════
 * Enum categories (unchanged from production contract)
 * ═══════════════════════════════════════════════════════════ */

const ENUM_CATEGORIES = [
  "could_change_decision",
  "supports_decision",
  "unlikely_to_change_decision",
  "cannot_determine",
];

/* ═══════════════════════════════════════════════════════════
 * Category definitions — identical to Experiment 52D and production
 * ═══════════════════════════════════════════════════════════ */

const CATEGORY_DEFINITIONS = {
  could_change_decision:
    "Answering could reasonably reverse the proposed action — it is a go/no-go condition or materially affects viability.",
  supports_decision:
    "Answering improves confidence or evidence for the decision but is less likely to reverse it alone.",
  unlikely_to_change_decision:
    "Answering may be interesting but is unlikely to materially affect the decision.",
  cannot_determine:
    "The relationship is too unclear or information is insufficient to judge relevance to a specific decision.",
};

/* ═══════════════════════════════════════════════════════════
 * Normalisation instruction — identical to Experiment 52D
 * ═══════════════════════════════════════════════════════════ */

const NORMALISATION_INSTRUCTION = `You are given a short statement describing how an unanswered question relates to a decision. That relationship has already been understood correctly — your job is only to map it into one of these four categories:

- "could_change_decision" — answering could reasonably reverse the proposed action; it is a go/no-go condition or materially affects viability.
- "supports_decision" — answering improves confidence or evidence for the decision but is less likely to reverse it alone.
- "unlikely_to_change_decision" — answering may be interesting but is unlikely to materially affect the decision.
- "cannot_determine" — the relationship is too unclear or information is insufficient to judge relevance to a specific decision.

Do not reinterpret the original situation — you have not been given it. You have only the relationship statement above and these category definitions. Choose the category that best matches the relationship statement.

Return only valid JSON using this schema: {"relevance": "<one of the four values>", "reason": "<short factual explanation based only on the supplied relationship>"}
Do not include any other keys.`;

/* ═══════════════════════════════════════════════════════════
 * Inline Ollama helper — one call per case, relationship-only input
 * ═══════════════════════════════════════════════════════════ */

function makeOllamaBody(instruction, relationship) {
  return JSON.stringify({
    model: process.env.OLLAMA_MODEL || "qwen-claude:latest",
    messages: [
      { role: "system", content: instruction },
      {
        role: "user",
        content: `Relationship: "${relationship}"`,
      },
    ],
    format: "json",
    stream: false,
  });
}

async function callBoundaryTest(relationship) {
  const baseUrl = process.env.OLLAMA_BASE_URL;
  if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");

  const model = process.env.OLLAMA_MODEL || "qwen-claude:latest";
  const body = makeOllamaBody(NORMALISATION_INSTRUCTION, relationship);
  const res = await fetch(`${baseUrl}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body,
    signal: AbortSignal.timeout(120000),
  });
  if (!res.ok) throw new Error(`Ollama returned ${res.status}`);
  const data = await res.json();
  const rawText =
    typeof data.message?.content === "string"
      ? data.message.content
      : JSON.stringify(data.message?.content || {});
  return { result: JSON.parse(rawText), model };
}

/* ═══════════════════════════════════════════════════════════
 * Three contrast pairs — six fixed relationship statements
 * Fixed before any model call. Expected enums decided before testing.
 * ═══════════════════════════════════════════════════════════ */

const CONTRAST_PAIRS = [
  {
    pair: "Pair 1 (Market Entry)",
    blocker: {
      id: "1A-blocker",
      relationship:
        "If the product cannot legally satisfy the required European regulations, entering the market cannot proceed.",
      expectedEnum: "could_change_decision",
    },
    supporting: {
      id: "1B-supporting",
      relationship:
        "Independent customer interviews showing strong interest would increase confidence that entering the European market is worthwhile, but would not determine the decision by themselves.",
      expectedEnum: "supports_decision",
    },
  },
  {
    pair: "Pair 2 (Community Event)",
    blocker: {
      id: "2A-blocker",
      relationship:
        "If the forecast shows dangerous weather conditions on the event date, holding the event outdoors would no longer be viable.",
      expectedEnum: "could_change_decision",
    },
    supporting: {
      id: "2B-supporting",
      relationship:
        "Positive feedback from previous attendees about outdoor events would strengthen confidence in choosing an outdoor venue, but would not decide the issue by itself.",
      expectedEnum: "supports_decision",
    },
  },
  {
    pair: "Pair 3 (Hiring Decision)",
    blocker: {
      id: "3A-blocker",
      relationship:
        "If the candidate does not hold the legally required professional licence, they cannot be appointed to the role.",
      expectedEnum: "could_change_decision",
    },
    supporting: {
      id: "3B-supporting",
      relationship:
        "Strong references from previous employers would increase confidence that the candidate is suitable, but would not determine the hiring decision alone.",
      expectedEnum: "supports_decision",
    },
  },
];

/* Flatten all six cases */
const ALL_CASES = [];
for (const pair of CONTRAST_PAIRS) {
  ALL_CASES.push({ ...pair.blocker, pairLabel: pair.pair });
  ALL_CASES.push({ ...pair.supporting, pairLabel: pair.pair });
}

/* ═══════════════════════════════════════════════════════════
 * Results holder — populated by beforeAll (6 calls total)
 * ═══════════════════════════════════════════════════════════ */

let experimentResults = {};
let inferenceCount = 0;
let timingStats = { min: Infinity, max: 0, total: 0 };
let modelFailureReason = null;

beforeAll(async () => {
  experimentResults = {};

  for (const c of ALL_CASES) {
    let result = null;
    let latency = 0;
    const t0 = Date.now();
    try {
      result = await callBoundaryTest(c.relationship);
      latency = Date.now() - t0;
    } catch (e) {
      modelFailureReason = `case ${c.id}: ${e.message}`;
      result = { result: null };
    }
    timingStats.min = Math.min(timingStats.min, latency);
    timingStats.max = Math.max(timingStats.max, latency);
    timingStats.total += latency;

    experimentResults[c.id] = {
      pair: c.pairLabel,
      relationship: c.relationship,
      expectedEnum: c.expectedEnum,
      returnedEnum: result.result?.relevance || "error",
      reason: result.result?.reason || "none",
      model: result.model,
      latencyMs: latency,
    };
    inferenceCount += 1;
  }
}, 600000);

/* ═══════════════════════════════════════════════════════════
 * Infrastructure assertions — exactly 6 calls, same config, production unchanged
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52E — Infrastructure", () => {
  it("uses Ollama config from .env.local", () => {
    expect(process.env.OLLAMA_BASE_URL).toBeTruthy();
    expect(process.env.OLLAMA_MODEL).toBe("qwen-claude:latest");
  });

  it("all returned enums belong to the existing four-category contract", () => {
    for (const c of ALL_CASES) {
      const r = experimentResults[c.id]?.returnedEnum;
      expect(ENUM_CATEGORIES).toContain(r);
    }
  });

  it("all cases include a reason string", () => {
    for (const c of ALL_CASES) {
      const r = experimentResults[c.id]?.reason;
      expect(typeof r).toBe("string");
      expect(r.length).toBeGreaterThan(0);
    }
  });

  it("same Ollama host used throughout", () => {
    expect(process.env.OLLAMA_BASE_URL).toBe("http://192.168.1.111:11434");
  });

  it("same model (qwen-claude:latest) used throughout", () => {
    for (const c of ALL_CASES) {
      expect(experimentResults[c.id]?.model).toBe("qwen-claude:latest");
    }
  });

  it("exactly 6 live inference calls were made", () => {
    expect(inferenceCount).toBe(6);
  });

  it("normalisation instruction is identical for all six calls", () => {
    expect(typeof NORMALISATION_INSTRUCTION).toBe("string");
    expect(NORMALISATION_INSTRUCTION.length).toBeGreaterThan(0);
  });

  it("fixed expected enums before any live call — structure check", () => {
    // Verify the fixed expected values match our design: 3 blockers = could_change_decision, 3 supporting = supports_decision
    let blockerCount = 0;
    let supportingCount = 0;
    for (const c of ALL_CASES) {
      if (c.expectedEnum === "could_change_decision") blockerCount++;
      if (c.expectedEnum === "supports_decision") supportingCount++;
    }
    expect(blockerCount).toBe(3);
    expect(supportingCount).toBe(3);
  });

  it("only the relationship statement is supplied to each case", () => {
    for (const c of ALL_CASES) {
      expect(c.relationship).toBeTruthy();
      expect(typeof c.relationship).toBe("string");
      // No decision target or question keys in the test data
      expect(c).not.toHaveProperty("decisionTarget");
      expect(c).not.toHaveProperty("question");
    }
  });

  it("category definitions unchanged from production", () => {
    const expectedKeys = Object.keys(CATEGORY_DEFINITIONS);
    expect(expectedKeys).toContain("could_change_decision");
    expect(expectedKeys).toContain("supports_decision");
    expect(expectedKeys).toContain("unlikely_to_change_decision");
    expect(expectedKeys).toContain("cannot_determine");

    // Verify definitions match production wording
    expect(CATEGORY_DEFINITIONS.could_change_decision).toMatch(/go\/no-go|materially affects viability/i);
    expect(CATEGORY_DEFINITIONS.supports_decision).toMatch(/improves confidence|less likely to reverse/i);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Blocker results — all three should map to could_change_decision
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52E — Blocker classification (could_change_decision)", () => {
  it("Pair 1A blocker maps to could_change_decision", () => {
    const r = experimentResults["1A-blocker"];
    expect(r.returnedEnum).toBe(r.expectedEnum);
    expect(typeof r.reason).toBe("string");
    expect(r.reason.length).toBeGreaterThan(0);
  });

  it("Pair 2A blocker maps to could_change_decision", () => {
    const r = experimentResults["2A-blocker"];
    expect(r.returnedEnum).toBe(r.expectedEnum);
    expect(typeof r.reason).toBe("string");
    expect(r.reason.length).toBeGreaterThan(0);
  });

  it("Pair 3A blocker maps to could_change_decision", () => {
    const r = experimentResults["3A-blocker"];
    expect(r.returnedEnum).toBe(r.expectedEnum);
    expect(typeof r.reason).toBe("string");
    expect(r.reason.length).toBeGreaterThan(0);
  });

  it("all three blockers consistently map to could_change_decision", () => {
    const blockerResults = ALL_CASES.filter((c) => c.expectedEnum === "could_change_decision");
    for (const c of blockerResults) {
      expect(experimentResults[c.id].returnedEnum).toBe("could_change_decision");
    }
  });
});

/* ═══════════════════════════════════════════════════════════
 * Supporting-evidence results — all three should map to supports_decision
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52E — Supporting evidence classification (supports_decision)", () => {
  it("Pair 1B supporting maps to supports_decision", () => {
    const r = experimentResults["1B-supporting"];
    expect(r.returnedEnum).toBe(r.expectedEnum);
    expect(typeof r.reason).toBe("string");
    expect(r.reason.length).toBeGreaterThan(0);
  });

  it("Pair 2B supporting maps to supports_decision", () => {
    const r = experimentResults["2B-supporting"];
    expect(r.returnedEnum).toBe(r.expectedEnum);
    expect(typeof r.reason).toBe("string");
    expect(r.reason.length).toBeGreaterThan(0);
  });

  it("Pair 3B supporting maps to supports_decision", () => {
    const r = experimentResults["3B-supporting"];
    expect(r.returnedEnum).toBe(r.expectedEnum);
    expect(typeof r.reason).toBe("string");
    expect(r.reason.length).toBeGreaterThan(0);
  });

  it("all three supporting-evidence cases consistently map to supports_decision", () => {
    const supportingResults = ALL_CASES.filter((c) => c.expectedEnum === "supports_decision");
    for (const c of supportingResults) {
      expect(experimentResults[c.id].returnedEnum).toBe("supports_decision");
    }
  });
});

/* ═══════════════════════════════════════════════════════════
 * Contrast-pair consistency — each pair's blocker vs supporting must differ
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52E — Contrast-pair consistency", () => {
  it("Pair 1: blocker and supporting map to different categories", () => {
    const rA = experimentResults["1A-blocker"].returnedEnum;
    const rB = experimentResults["1B-supporting"].returnedEnum;
    expect(rA).not.toBe(rB);
  });

  it("Pair 2: blocker and supporting map to different categories", () => {
    const rA = experimentResults["2A-blocker"].returnedEnum;
    const rB = experimentResults["2B-supporting"].returnedEnum;
    expect(rA).not.toBe(rB);
  });

  it("Pair 3: blocker and supporting map to different categories", () => {
    const rA = experimentResults["3A-blocker"].returnedEnum;
    const rB = experimentResults["3B-supporting"].returnedEnum;
    expect(rA).not.toBe(rB);
  });

  it("no supporting evidence was classified as could_change_decision (decision-reverser misclassification)", () => {
    const supportingCases = ALL_CASES.filter((c) => c.expectedEnum === "supports_decision");
    for (const c of supportingCases) {
      expect(experimentResults[c.id].returnedEnum).not.toBe("could_change_decision");
    }
  });

  it("no blocker was classified as supports_decision (blocker downgraded to supportive)", () => {
    const blockerCases = ALL_CASES.filter((c) => c.expectedEnum === "could_change_decision");
    for (const c of blockerCases) {
      expect(experimentResults[c.id].returnedEnum).not.toBe("supports_decision");
    }
  });

  it("full output log", () => {
    for (const c of ALL_CASES) {
      const r = experimentResults[c.id];
      const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
      console.log(
        `[${r.pair}] ${c.id}: expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match} | ` +
          `reason="${r.reason}" | latency=${r.latencyMs}ms`
      );
    }
  });
});

/* ═══════════════════════════════════════════════════════════
 * Inference timing (observational only)
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52E — Inference timing", () => {
  it("records min, max, total timing for all 6 calls", () => {
    expect(timingStats.min).toBeGreaterThan(0);
    expect(timingStats.max).toBeGreaterThanOrEqual(timingStats.min);
    expect(timingStats.total).toBeGreaterThan(0);
  });

  it("records average latency within reasonable range", () => {
    const avg = timingStats.total / 6;
    expect(avg).toBeGreaterThan(5000);
    expect(avg).toBeLessThan(120000);
  });

  it("logs timing summary", () => {
    const avg = Math.round(timingStats.total / 6);
    console.log(`\n=== Experiment 52E Timing ===`);
    console.log(`Calls: 6`);
    console.log(`Total: ${timingStats.total}ms`);
    console.log(`Average: ${avg}ms`);
    console.log(`Fastest: ${timingStats.min}ms`);
    console.log(`Slowest: ${timingStats.max}ms`);
  });
});