confidence-engine/tests/graph/decision-relevance-domain-priors.test.js

/**
 * Experiment 52G — Does the Model Fill Ambiguous Meaning With Domain Expectations?
 *
 * Passive ambiguity/generalisation experiment. Tests whether ambiguous relationship statements
 * are pushed toward different stronger categories depending on what the model believes matters
 * in each domain, or whether they all remain `cannot_determine`.
 *
 * Four fixed ambiguous relationship statements with identical linguistic structure across four
 * different domains. Uses exactly the same category definitions and normalisation instruction
 * as Experiment 52F. Does not change any production code, category definitions, classifier,
 * or active engine.
 *
 * Hypothesis: If the model generally fills semantic gaps from domain knowledge (not just
 * regulatory priors), then different domains will produce different stronger categories despite
 * having the same degree of explicitness in their statements.
 */

import dotenv from "dotenv";
dotenv.config({ path: ".env.local" });

import { describe, it, expect, beforeAll } from "vitest";

/* ═══════════════════════════════════════════════════════════
 * Enum categories (unchanged from production contract)
 * ═══════════════════════════════════════════════════════════ */

const ENUM_CATEGORIES = [
  "could_change_decision",
  "supports_decision",
  "unlikely_to_change_decision",
  "cannot_determine",
];

/* ═══════════════════════════════════════════════════════════
 * Category definitions — identical to Experiment 52F and production
 * ═══════════════════════════════════════════════════════════ */

const CATEGORY_DEFINITIONS = {
  could_change_decision:
    "Answering could reasonably reverse the proposed action — it is a go/no-go condition or materially affects viability.",
  supports_decision:
    "Answering improves confidence or evidence for the decision but is less likely to reverse it alone.",
  unlikely_to_change_decision:
    "Answering may be interesting but is unlikely to materially affect the decision.",
  cannot_determine:
    "The relationship is too unclear or information is insufficient to judge relevance to a specific decision.",
};

/* ═══════════════════════════════════════════════════════════
 * Normalisation instruction — identical to Experiment 52F (unchanged)
 * ═══════════════════════════════════════════════════════════ */

const NORMALISATION_INSTRUCTION = `You are given a short statement describing how an unanswered question relates to a decision. That relationship has already been understood correctly — your job is only to map it into one of these four categories:

- "could_change_decision" — answering could reasonably reverse the proposed action; it is a go/no-go condition or materially affects viability.
- "supports_decision" — answering improves confidence or evidence for the decision but is less likely to reverse it alone.
- "unlikely_to_change_decision" — answering may be interesting but is unlikely to materially affect the decision.
- "cannot_determine" — the relationship is too unclear or information is insufficient to judge relevance to a specific decision.

Do not reinterpret the original situation — you have not been given it. You have only the relationship statement above and these category definitions. Choose the category that best matches the relationship statement.

Return only valid JSON using this schema: {"relevance": "<one of the four values>", "reason": "<short factual explanation based only on the supplied relationship>"}
Do not include any other keys.`;

/* ═══════════════════════════════════════════════════════════
 * Inline Ollama helper — one call per case, relationship-only input
 * ═══════════════════════════════════════════════════════════ */

function makeOllamaBody(instruction, relationship) {
  return JSON.stringify({
    model: process.env.OLLAMA_MODEL || "qwen-claude:latest",
    messages: [
      { role: "system", content: instruction },
      {
        role: "user",
        content: `Relationship: "${relationship}"`,
      },
    ],
    format: "json",
    stream: false,
  });
}

async function callDomainPriorTest(relationship) {
  const baseUrl = process.env.OLLAMA_BASE_URL;
  if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");

  const model = process.env.OLLAMA_MODEL || "qwen-claude:latest";
  const body = makeOllamaBody(NORMALISATION_INSTRUCTION, relationship);
  const res = await fetch(`${baseUrl}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body,
    signal: AbortSignal.timeout(120000),
  });
  if (!res.ok) throw new Error(`Ollama returned ${res.status}`);
  const data = await res.json();
  const rawText =
    typeof data.message?.content === "string"
      ? data.message.content
      : JSON.stringify(data.message?.content || {});
  return { result: JSON.parse(rawText), model };
}

/* ═══════════════════════════════════════════════════════════
 * Four structurally matched ambiguous relationship statements
 * Fixed before any model call. Expected enum is `cannot_determine` for all.
 *
 * Each uses the same linguistic template:
 *   "Understanding [X] would be important to [decision]."
 * ═══════════════════════════════════════════════════════════ */

const CASES = [
  {
    id: "case1-regulation",
    domain: "regulation",
    description: "Regulation",
    relationship:
      "Understanding the regulatory position would be important to the market-entry decision.",
    expectedEnum: "cannot_determine",
  },
  {
    id: "case2-weather",
    domain: "weather",
    description: "Weather",
    relationship:
      "Understanding the weather outlook would be important to the outdoor-event decision.",
    expectedEnum: "cannot_determine",
  },
  {
    id: "case3-employment-references",
    domain: "employment",
    description: "Employment References",
    relationship:
      "Understanding what the candidate's references say would be important to the hiring decision.",
    expectedEnum: "cannot_determine",
  },
  {
    id: "case4-customer-feedback",
    domain: "customer-feedback",
    description: "Customer Feedback",
    relationship:
      "Understanding what customers think would be important to the product-launch decision.",
    expectedEnum: "cannot_determine",
  },
];

/* ═══════════════════════════════════════════════════════════
 * External-assumption diagnostic labels
 * ═══════════════════════════════════════════════════════════ */

const GROUNDING_LABELS = {
  GROUNDED_ONLY: "grounded_only_in_statement",
  EXTERNAL_ASSUMPTION: "introduced_external_assumption",
};

/* ═══════════════════════════════════════════════════════════
 * Results holder — populated by beforeAll (4 calls total)
 * ═══════════════════════════════════════════════════════════ */

let experimentResults = {};
let inferenceCount = 0;
let timingStats = { min: Infinity, max: 0, total: 0 };
let modelFailureReason = null;

beforeAll(async () => {
  experimentResults = {};

  for (const c of CASES) {
    let result = null;
    let latency = 0;
    const t0 = Date.now();
    try {
      result = await callDomainPriorTest(c.relationship);
      latency = Date.now() - t0;
    } catch (e) {
      modelFailureReason = `case ${c.id}: ${e.message}`;
      result = { result: null };
    }
    timingStats.min = Math.min(timingStats.min, latency);
    timingStats.max = Math.max(timingStats.max, latency);
    timingStats.total += latency;

    experimentResults[c.id] = {
      domain: c.domain,
      description: c.description,
      relationship: c.relationship,
      expectedEnum: c.expectedEnum,
      returnedEnum: result.result?.relevance || "error",
      reason: result.result?.reason || "none",
      model: result.model,
      latencyMs: latency,
    };
    inferenceCount += 1;
  }
}, 600000);

/* ═══════════════════════════════════════════════════════════
 * Infrastructure assertions — exactly 4 calls, same config, production unchanged
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52G — Infrastructure", () => {
  it("uses Ollama config from .env.local", () => {
    expect(process.env.OLLAMA_BASE_URL).toBeTruthy();
    expect(process.env.OLLAMA_MODEL).toBe("qwen-claude:latest");
  });

  it("all returned enums belong to the existing four-category contract", () => {
    for (const c of CASES) {
      const r = experimentResults[c.id]?.returnedEnum;
      expect(ENUM_CATEGORIES).toContain(r);
    }
  });

  it("all cases include a reason string", () => {
    for (const c of CASES) {
      const r = experimentResults[c.id]?.reason;
      expect(typeof r).toBe("string");
      expect(r.length).toBeGreaterThan(0);
    }
  });

  it("same Ollama host used throughout", () => {
    expect(process.env.OLLAMA_BASE_URL).toBe("http://192.168.1.111:11434");
  });

  it("same model (qwen-claude:latest) used throughout", () => {
    for (const c of CASES) {
      expect(experimentResults[c.id]?.model).toBe("qwen-claude:latest");
    }
  });

  it("exactly 4 live inference calls were made", () => {
    expect(inferenceCount).toBe(4);
  });

  it("normalisation instruction is identical for all four calls", () => {
    expect(typeof NORMALISATION_INSTRUCTION).toBe("string");
    expect(NORMALISATION_INSTRUCTION.length).toBeGreaterThan(0);
  });

  it("fixed expected enums before any live call — all four are cannot_determine", () => {
    let ambiguousCount = 0;
    for (const c of CASES) {
      if (c.expectedEnum === "cannot_determine") ambiguousCount++;
    }
    expect(ambiguousCount).toBe(4);
  });

  it("all four cases use closely matched linguistic structure", () => {
    const patterns = CASES.map((c) => c.relationship);
    // All should contain "Understanding" and "would be important to"
    for (const p of patterns) {
      expect(p).toMatch(/Understanding.*would be important to/i);
      expect(p).toMatch(/decision\.$/i);
    }
  });

  it("only the relationship statement is supplied to each case", () => {
    for (const c of CASES) {
      expect(c.relationship).toBeTruthy();
      expect(typeof c.relationship).toBe("string");
      expect(c).not.toHaveProperty("decisionTarget");
      expect(c).not.toHaveProperty("question");
    }
  });

  it("category definitions unchanged from production", () => {
    const expectedKeys = Object.keys(CATEGORY_DEFINITIONS);
    expect(expectedKeys).toContain("could_change_decision");
    expect(expectedKeys).toContain("supports_decision");
    expect(expectedKeys).toContain("unlikely_to_change_decision");
    expect(expectedKeys).toContain("cannot_determine");

    expect(CATEGORY_DEFINITIONS.could_change_decision).toMatch(/go\/no-go|materially affects viability/i);
    expect(CATEGORY_DEFINITIONS.supports_decision).toMatch(/improves confidence|less likely to reverse/i);
  });

  it("instruction and definitions unchanged from Experiment 52F", () => {
    const expectedInstruction = `You are given a short statement describing how an unanswered question relates to a decision. That relationship has already been understood correctly — your job is only to map it into one of these four categories:

- "could_change_decision" — answering could reasonably reverse the proposed action; it is a go/no-go condition or materially affects viability.
- "supports_decision" — answering improves confidence or evidence for the decision but is less likely to reverse it alone.
- "unlikely_to_change_decision" — answering may be interesting but is unlikely to materially affect the decision.
- "cannot_determine" — the relationship is too unclear or information is insufficient to judge relevance to a specific decision.

Do not reinterpret the original situation — you have not been given it. You have only the relationship statement above and these category definitions. Choose the category that best matches the relationship statement.

Return only valid JSON using this schema: {"relevance": "<one of the four values>", "reason": "<short factual explanation based only on the supplied relationship>"}
Do not include any other keys.`;
    expect(NORMALISATION_INSTRUCTION).toBe(expectedInstruction);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Case 1 — Regulation
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52G — Case 1: Regulation", () => {
  it("returns its enum result within the contract", () => {
    const r = experimentResults["case1-regulation"];
    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
    expect(typeof r.reason).toBe("string");
    expect(r.reason.length).toBeGreaterThan(0);
  });

  it("expected enum is cannot_determine", () => {
    const r = experimentResults["case1-regulation"];
    expect(r.expectedEnum).toBe("cannot_determine");
  });

  it("match/mismatch check", () => {
    const r = experimentResults["case1-regulation"];
    const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
    console.log(`[Regulation] expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match}`);
  });

  it("external-assumption diagnostic", () => {
    const r = experimentResults["case1-regulation"];
    // Check for regulation-specific external assumptions (treats "important" as go/no-go blocker)
    const reasonLower = r.reason.toLowerCase();
    const hasBlockerAssumption = /regulatory.*always|compliance.*mandatory|regulation.*must.*block|legal.*prerequisite|cannot proceed without|legally mandatory|by definition.*blocking/i.test(reasonLower);
    const grounding = hasBlockerAssumption ? "introduced_external_assumption" : "grounded_only_in_statement";
    console.log(`[Regulation] external-assumption: ${grounding}`);
    expect(grounding).toBeTruthy();
  });
});

/* ═══════════════════════════════════════════════════════════
 * Case 2 — Weather
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52G — Case 2: Weather", () => {
  it("returns its enum result within the contract", () => {
    const r = experimentResults["case2-weather"];
    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
    expect(typeof r.reason).toBe("string");
    expect(r.reason.length).toBeGreaterThan(0);
  });

  it("expected enum is cannot_determine", () => {
    const r = experimentResults["case2-weather"];
    expect(r.expectedEnum).toBe("cannot_determine");
  });

  it("match/mismatch check", () => {
    const r = experimentResults["case2-weather"];
    const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
    console.log(`[Weather] expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match}`);
  });

  it("external-assumption diagnostic", () => {
    const r = experimentResults["case2-weather"];
    // Check for weather-specific external assumptions (treats "important" as cancellation risk)
    const reasonLower = r.reason.toLowerCase();
    const hasBlockerAssumption = /weather.*automatically|bad weather.*impossible|dangerous.*cancels|extreme.*force.*cancel|necessarily.*prevent/i.test(reasonLower);
    const grounding = hasBlockerAssumption ? "introduced_external_assumption" : "grounded_only_in_statement";
    console.log(`[Weather] external-assumption: ${grounding}`);
    expect(grounding).toBeTruthy();
  });
});

/* ═══════════════════════════════════════════════════════════
 * Case 3 — Employment References
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52G — Case 3: Employment References", () => {
  it("returns its enum result within the contract", () => {
    const r = experimentResults["case3-employment-references"];
    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
    expect(typeof r.reason).toBe("string");
    expect(r.reason.length).toBeGreaterThan(0);
  });

  it("expected enum is cannot_determine", () => {
    const r = experimentResults["case3-employment-references"];
    expect(r.expectedEnum).toBe("cannot_determine");
  });

  it("match/mismatch check", () => {
    const r = experimentResults["case3-employment-references"];
    const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
    console.log(`[Employment References] expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match}`);
  });

  it("external-assumption diagnostic", () => {
    const r = experimentResults["case3-employment-references"];
    // Check for employment-specific external assumptions (treats "important" as decisive)
    const reasonLower = r.reason.toLowerCase();
    const hasBlockerAssumption = /references.*determine|hiring decision depends|mandatory.*reference|must.*pass reference|reference.*disqualify|automatically.*suitable/i.test(reasonLower);
    const grounding = hasBlockerAssumption ? "introduced_external_assumption" : "grounded_only_in_statement";
    console.log(`[Employment References] external-assumption: ${grounding}`);
    expect(grounding).toBeTruthy();
  });
});

/* ═══════════════════════════════════════════════════════════
 * Case 4 — Customer Feedback
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52G — Case 4: Customer Feedback", () => {
  it("returns its enum result within the contract", () => {
    const r = experimentResults["case4-customer-feedback"];
    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
    expect(typeof r.reason).toBe("string");
    expect(r.reason.length).toBeGreaterThan(0);
  });

  it("expected enum is cannot_determine", () => {
    const r = experimentResults["case4-customer-feedback"];
    expect(r.expectedEnum).toBe("cannot_determine");
  });

  it("match/mismatch check", () => {
    const r = experimentResults["case4-customer-feedback"];
    const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
    console.log(`[Customer Feedback] expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match}`);
  });

  it("external-assumption diagnostic", () => {
    const r = experimentResults["case4-customer-feedback"];
    // Check for customer-feedback-specific external assumptions (treats "important" as viability blocker)
    const reasonLower = r.reason.toLowerCase();
    const hasBlockerAssumption = /customer.*stop.*launch|feedback.*determines.*viability|must.*cancel.*product|necessarily.*block.*launch/i.test(reasonLower);
    const grounding = hasBlockerAssumption ? "introduced_external_assumption" : "grounded_only_in_statement";
    console.log(`[Customer Feedback] external-assumption: ${grounding}`);
    expect(grounding).toBeTruthy();
  });
});

/* ═══════════════════════════════════════════════════════════
 * Cross-domain comparison — does domain affect category choice?
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52G — Cross-domain analysis", () => {
  it("reports all matches and mismatches", () => {
    for (const c of CASES) {
      const r = experimentResults[c.id];
      const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
      console.log(
        `[${c.description}] ${c.domain}: expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match} | ` +
          `reason="${r.reason}" | latency=${r.latencyMs}ms`
      );
    }
  });

  it("cannot_determine count for all four cases", () => {
    let canNotDetermineCount = 0;
    for (const c of CASES) {
      const r = experimentResults[c.id];
      if (r.returnedEnum === "cannot_determine") canNotDetermineCount++;
    }
    console.log(`Cannot_determine count: ${canNotDetermineCount}/4`);
    expect(typeof canNotDetermineCount).toBe("number");
  });

  it("does regulation return could_change_decision again?", () => {
    const r = experimentResults["case1-regulation"];
    const isCouldChange = r.returnedEnum === "could_change_decision";
    console.log(`[Regulation] returned could_change_decision: ${isCouldChange}`);
    expect(typeof isCouldChange).toBe("boolean");
  });

  it("did different domains produce different stronger categories?", () => {
    const categories = CASES.map((c) => experimentResults[c.id].returnedEnum);
    const uniqueCategories = [...new Set(categories)];
    const hadDivergence = uniqueCategories.length > 1;
    console.log(`Unique categories across domains: ${uniqueCategories.join(", ")} | divergence: ${hadDivergence}`);
    expect(typeof hadDivergence).toBe("boolean");
  });

  it("count of cases where model introduced external assumptions", () => {
    let assumedCount = 0;
    for (const c of CASES) {
      const r = experimentResults[c.id];
      if (r.expectedEnum === "cannot_determine") {
        const reasonLower = r.reason.toLowerCase();
        const hasAssumption = (() => {
          switch (c.domain) {
            case "regulation":
              return /regulatory.*always|compliance.*mandatory|regulation.*must.*block|legal.*prerequisite|cannot proceed without|legally mandatory|by definition.*blocking/i.test(reasonLower);
            case "weather":
              return /weather.*automatically|bad weather.*impossible|dangerous.*cancels|extreme.*force.*cancel|necessarily.*prevent/i.test(reasonLower);
            case "employment":
              return /references.*determine|hiring decision depends|mandatory.*reference|must.*pass reference|reference.*disqualify|automatically.*suitable/i.test(reasonLower);
            case "customer-feedback":
              return /customer.*stop.*launch|feedback.*determines.*viability|must.*cancel.*product|necessarily.*block.*launch/i.test(reasonLower);
            default:
              return false;
          }
        })();
        if (hasAssumption) assumedCount++;
      }
    }
    console.log(`Cases with external assumptions: ${assumedCount}/4`);
    expect(typeof assumedCount).toBe("number");
  });

  it("full output log", () => {
    console.log("\n=== Experiment 52G Summary ===");
    for (const c of CASES) {
      const r = experimentResults[c.id];
      const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
      const reasonLower = r.reason.toLowerCase();
      let grounding = "grounded_only_in_statement";
      if (r.expectedEnum === "cannot_determine") {
        const hasAssumption = (() => {
          switch (c.domain) {
            case "regulation":
              return /regulatory.*always|compliance.*mandatory|regulation.*must.*block|legal.*prerequisite|cannot proceed without|legally mandatory|by definition.*blocking/i.test(reasonLower);
            case "weather":
              return /weather.*automatically|bad weather.*impossible|dangerous.*cancels|extreme.*force.*cancel|necessarily.*prevent/i.test(reasonLower);
            case "employment":
              return /references.*determine|hiring decision depends|mandatory.*reference|must.*pass reference|reference.*disqualify|automatically.*suitable/i.test(reasonLower);
            case "customer-feedback":
              return /customer.*stop.*launch|feedback.*determines.*viability|must.*cancel.*product|necessarily.*block.*launch/i.test(reasonLower);
            default:
              return false;
          }
        })();
        if (hasAssumption) grounding = "introduced_external_assumption";
      }
      console.log(
        `[${c.description}] ${r.expectedEnum} → ${r.returnedEnum} (${match}) | ` +
          `reason="${r.reason}" | domain_assumption=${grounding} | latency=${r.latencyMs}ms`
      );
    }
  });
});

/* ═══════════════════════════════════════════════════════════
 * Inference timing (observational only)
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52G — Inference timing", () => {
  it("records min, max, total timing for all 4 calls", () => {
    expect(timingStats.min).toBeGreaterThan(0);
    expect(timingStats.max).toBeGreaterThanOrEqual(timingStats.min);
    expect(timingStats.total).toBeGreaterThan(0);
  });

  it("records average latency within reasonable range", () => {
    const avg = timingStats.total / 4;
    expect(avg).toBeGreaterThan(5000);
    expect(avg).toBeLessThan(120000);
  });

  it("logs timing summary", () => {
    const avg = Math.round(timingStats.total / 4);
    console.log(`\n=== Experiment 52G Timing ===`);
    console.log(`Calls: 4`);
    console.log(`Total: ${timingStats.total}ms`);
    console.log(`Average: ${avg}ms`);
    console.log(`Fastest: ${timingStats.min}ms`);
    console.log(`Slowest: ${timingStats.max}ms`);
  });
});