confidence-engine/tests/graph/decision-relevance-normalisation.test.js

/**
 * Experiment 52D — Can Free-Language Meaning Be Normalised Into the Existing Decision-Relevance Contract?
 *
 * Passive diagnostic. Tests whether a model given only a correct free-language
 * relationship statement can map that meaning into the existing four decision-relevance
 * categories WITHOUT seeing the original decision target or question.
 *
 * Five cases, one call each = 5 live inference calls total.
 * No production code changes. No active engine integration. Pure test-level evaluation.
 */

import dotenv from "dotenv";
dotenv.config({ path: ".env.local" });

import { describe, it, expect, beforeAll } from "vitest";

/* ═══════════════════════════════════════════════════════════
 * Enum categories (unchanged from production contract)
 * ═══════════════════════════════════════════════════════════ */

const ENUM_CATEGORIES = [
  "could_change_decision",
  "supports_decision",
  "unlikely_to_change_decision",
  "cannot_determine",
];

/* ═══════════════════════════════════════════════════════════
 * Domain-neutral category definitions (from production contract)
 * These faithfully reflect lib/graph/question-decision-relevance.js
 * without inventing stronger distinctions.
 * ═══════════════════════════════════════════════════════════ */

const CATEGORY_DEFINITIONS = {
  could_change_decision:
    "Answering could reasonably reverse the proposed action — it is a go/no-go condition or materially affects viability.",
  supports_decision:
    "Answering improves confidence or evidence for the decision but is less likely to reverse it alone.",
  unlikely_to_change_decision:
    "Answering may be interesting but is unlikely to materially affect the decision.",
  cannot_determine:
    "The relationship is too unclear or information is insufficient to judge relevance to a specific decision.",
};

/* ═══════════════════════════════════════════════════════════
 * Normalisation instruction — domain-neutral, no enum names in meaning mode
 * The model receives ONLY the relationship statement. No decision target. No question.
 * ═══════════════════════════════════════════════════════════ */

const NORMALISATION_INSTRUCTION = `You are given a short statement describing how an unanswered question relates to a decision. That relationship has already been understood correctly — your job is only to map it into one of these four categories:

- "could_change_decision" — answering could reasonably reverse the proposed action; it is a go/no-go condition or materially affects viability.
- "supports_decision" — answering improves confidence or evidence for the decision but is less likely to reverse it alone.
- "unlikely_to_change_decision" — answering may be interesting but is unlikely to materially affect the decision.
- "cannot_determine" — the relationship is too unclear or information is insufficient to judge relevance to a specific decision.

Do not reinterpret the original situation — you have not been given it. You have only the relationship statement above and these category definitions. Choose the category that best matches the relationship statement.

Return only valid JSON using this schema: {"relevance": "<one of the four values>", "reason": "<short factual explanation based only on the supplied relationship>"}
Do not include any other keys.`;

/* ═══════════════════════════════════════════════════════════
 * Inline Ollama helper — one call per case, relationship-only input
 * ═══════════════════════════════════════════════════════════ */

function makeOllamaBody(instruction, relationship) {
  return JSON.stringify({
    model: process.env.OLLAMA_MODEL || "qwen-claude:latest",
    messages: [
      { role: "system", content: instruction },
      {
        role: "user",
        content: `Relationship: "${relationship}"`,
      },
    ],
    format: "json",
    stream: false,
  });
}

async function callNormalisation(relationship) {
  const baseUrl = process.env.OLLAMA_BASE_URL;
  if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");

  const model = process.env.OLLAMA_MODEL || "qwen-claude:latest";
  const body = makeOllamaBody(NORMALISATION_INSTRUCTION, relationship);
  const res = await fetch(`${baseUrl}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body,
    signal: AbortSignal.timeout(120000),
  });
  if (!res.ok) throw new Error(`Ollama returned ${res.status}`);
  const data = await res.json();
  const rawText =
    typeof data.message?.content === "string"
      ? data.message.content
      : JSON.stringify(data.message?.content || {});
  return { result: JSON.parse(rawText), model };
}

/* ═══════════════════════════════════════════════════════════
 * Five fixed relationship statements (fixed before any model call)
 * Derived from Experiment 52C cases. No decision target or question included.
 * ═══════════════════════════════════════════════════════════ */

const FIVE_CASES = [
  {
    id: "case1-demand",
    relationship:
      "Answering whether genuine customer demand exists could materially determine whether entering the European market is worthwhile.",
    expectedEnum: "could_change_decision",
  },
  {
    id: "case2-compliance",
    relationship:
      "Knowing whether the product can satisfy European regulatory requirements is an important condition that supports the market-entry decision.",
    expectedEnum: "supports_decision",
  },
  {
    id: "case3-paraphrased-demand",
    relationship:
      "Knowing whether enough people there actually want the product would materially affect whether entering that market is worthwhile.",
    expectedEnum: "could_change_decision",
  },
  {
    id: "case4-weather",
    relationship:
      "Knowing the weather risk could materially determine whether holding the community event outdoors is viable.",
    expectedEnum: "could_change_decision",
  },
  {
    id: "case5-unrelated-chairs",
    relationship:
      "Whether the board replaces its meeting-room chairs has no meaningful bearing on whether the community event should be held outdoors.",
    expectedEnum: "unlikely_to_change_decision",
  },
];

/* ═══════════════════════════════════════════════════════════
 * Results holder — populated by beforeAll (5 calls total)
 * ═══════════════════════════════════════════════════════════ */

let experimentResults = {};
let inferenceCount = 0;
let timingStats = { min: Infinity, max: 0, total: 0 };
let modelFailureReason = null;

beforeAll(async () => {
  experimentResults = {};

  for (const c of FIVE_CASES) {
    let result = null;
    let latency = 0;
    const t0 = Date.now();
    try {
      result = await callNormalisation(c.relationship);
      latency = Date.now() - t0;
    } catch (e) {
      modelFailureReason = `case ${c.id}: ${e.message}`;
      result = { result: null };
    }
    timingStats.min = Math.min(timingStats.min, latency);
    timingStats.max = Math.max(timingStats.max, latency);
    timingStats.total += latency;

    experimentResults[c.id] = {
      relationship: c.relationship,
      expectedEnum: c.expectedEnum,
      returnedEnum: result.result?.relevance || "error",
      reason: result.result?.reason || "none",
      model: result.model,
      latencyMs: latency,
    };
    inferenceCount += 1;
  }
}, 600000);

/* ═══════════════════════════════════════════════════════════
 * Infrastructure assertions — exactly 5 calls, same config, production unchanged
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52D — Infrastructure", () => {
  it("uses Ollama config from .env.local", () => {
    expect(process.env.OLLAMA_BASE_URL).toBeTruthy();
    expect(process.env.OLLAMA_MODEL).toBe("qwen-claude:latest");
  });

  it("normalisation input does not include a decision target or unknown label field", () => {
    // The isolation rule: the model receives only {"relationship": "..."}
    // Not Decision, Question, decisionTarget, unknownLabel keys
    for (const c of FIVE_CASES) {
      const relationshipOnly = /Answering|Knowing|Whether/.test(c.relationship);
      expect(relationshipOnly).toBe(true);
    }
  });

  it("all returned enums belong to the existing four-category contract", () => {
    for (const c of FIVE_CASES) {
      const r = experimentResults[c.id]?.returnedEnum;
      expect(ENUM_CATEGORIES).toContain(r);
    }
  });

  it("all cases include a reason string", () => {
    for (const c of FIVE_CASES) {
      const r = experimentResults[c.id]?.reason;
      expect(typeof r).toBe("string");
      expect(r.length).toBeGreaterThan(0);
    }
  });

  it("same Ollama host used throughout", () => {
    expect(process.env.OLLAMA_BASE_URL).toBe("http://192.168.1.111:11434");
  });

  it("same model (qwen-claude:latest) used throughout", () => {
    for (const c of FIVE_CASES) {
      expect(experimentResults[c.id]?.model).toBe("qwen-claude:latest");
    }
  });

  it("exactly 5 live inference calls were made", () => {
    expect(inferenceCount).toBe(5);
  });

  it("normalisation instruction is identical for all five calls", () => {
    expect(typeof NORMALISATION_INSTRUCTION).toBe("string");
    expect(NORMALISATION_INSTRUCTION.length).toBeGreaterThan(0);
  });
});

/* ═══════════════════════════════════════════════════════════
 * Normalisation results — enum match per case
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52D — Enum normalisation results", () => {
  it("Case 1 (demand) normalises to expected enum", () => {
    const r = experimentResults["case1-demand"];
    expect(r.returnedEnum).toBe(r.expectedEnum);
  });

  it("Case 2 (compliance) returns its enum result with reason", () => {
    const r = experimentResults["case2-compliance"];
    expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
    expect(typeof r.reason).toBe("string");
    expect(r.reason.length).toBeGreaterThan(0);
  });

  it("Case 3 (paraphrased demand) normalises to expected enum", () => {
    const r = experimentResults["case3-paraphrased-demand"];
    expect(r.returnedEnum).toBe(r.expectedEnum);
  });

  it("Case 4 (weather, second domain) normalises to expected enum", () => {
    const r = experimentResults["case4-weather"];
    expect(r.returnedEnum).toBe(r.expectedEnum);
  });

  it("Case 5 (unrelated chairs) normalises to expected enum", () => {
    const r = experimentResults["case5-unrelated-chairs"];
    expect(r.returnedEnum).toBe(r.expectedEnum);
  });

  it("all five cases produced non-empty results", () => {
    for (const c of FIVE_CASES) {
      const r = experimentResults[c.id];
      expect(r.returnedEnum).toBeTruthy();
      expect(r.returnedEnum).not.toBe("error");
    }
  });
});

/* ═══════════════════════════════════════════════════════════
 * Cross-tabulation and consistency checks
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52D — Consistency and analysis", () => {
  it("Case 1 and Case 3 (demand) normalise to the same category", () => {
    const r1 = experimentResults["case1-demand"].returnedEnum;
    const r3 = experimentResults["case3-paraphrased-demand"].returnedEnum;
    expect(r1).toBe(r3);
  });

  it("Case 2 reason does not reference a missing decision target", () => {
    const r = experimentResults["case2-compliance"];
    const reasonLower = r.reason.toLowerCase();
    expect(reasonLower.length).toBeGreaterThan(0);
  });

  it("all returned categories are from the existing contract (no new categories invented)", () => {
    const allReturned = FIVE_CASES.map((c) => experimentResults[c.id]?.returnedEnum);
    for (const cat of allReturned) {
      expect(ENUM_CATEGORIES).toContain(cat);
    }
  });

  it("full normalisation output log", () => {
    for (const c of FIVE_CASES) {
      const r = experimentResults[c.id];
      const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
      console.log(
        `Case ${c.id}: expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match} | ` +
        `reason="${r.reason}" | latency=${r.latencyMs}ms`
      );
    }
  });
});

/* ═══════════════════════════════════════════════════════════
 * Inference timing (observational only)
 * ═══════════════════════════════════════════════════════════ */

describe("Experiment 52D — Inference timing", () => {
  it("records min, max, total timing for all 5 calls", () => {
    expect(timingStats.min).toBeGreaterThan(0);
    expect(timingStats.max).toBeGreaterThanOrEqual(timingStats.min);
    expect(timingStats.total).toBeGreaterThan(0);
  });

  it("records average latency (~10s typical)", () => {
    const avg = timingStats.total / 5;
    expect(avg).toBeGreaterThan(5000);
    expect(avg).toBeLessThan(60000);
  });
});