confidence-engine/tests/diagnostic/live-diagnostic.test.js

import { describe, it, expect } from "vitest";
import { readFileSync, existsSync, readdirSync } from "node:fs";
import { join, dirname } from "node:path";
import { fileURLToPath } from "node:url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const rootDir = join(__dirname, "..", "..");

// ── Test data loading and structure ────────────────

describe("live-diagnostic test data", () => {
  const cases = JSON.parse(
    readFileSync(join(__dirname, "data", "live-diagnostic-v0.2.json"), "utf-8")
  );

  it("loads without error", () => {
    expect(cases).toBeDefined();
    expect(Array.isArray(cases)).toBe(true);
  });

  it("contains exactly 10 cases", () => {
    expect(cases.length).toBe(10);
  });

  it("each case has required fields (id, input, expectedPrimaryTypes)", () => {
    for (const c of cases) {
      expect(c.id).toBeDefined();
      expect(typeof c.id).toBe("string");
      expect(c.input).toBeDefined();
      expect(typeof c.input).toBe("string");
      expect(c.input.length).toBeGreaterThan(0);
      expect(c.expectedPrimaryTypes).toBeDefined();
      expect(Array.isArray(c.expectedPrimaryTypes)).toBe(true);
      expect(c.shouldIdentify).toBeDefined();
      expect(c.shouldNotInfer).toBeDefined();
    }
  });

  it("has unique case IDs", () => {
    const ids = cases.map((c) => c.id);
    const uniqueIds = new Set(ids);
    expect(uniqueIds.size).toBe(ids.length);
  });

  it("IDs follow diag-NN naming convention", () => {
    const ids = cases.map((c) => c.id);
    for (const id of ids) {
      expect(id).toMatch(/^diag-\d{2}$/);
    }
  });

  it("has no duplicate shouldIdentify/shouldNotInfer sets (paired cases differ)", () => {
    // diag-01 and diag-10 are the "paired" cases — they share context but not identical assertions
    const diag01 = cases.find((c) => c.id === "diag-01");
    const diag10 = cases.find((c) => c.id === "diag-10");
    expect(diag01).toBeDefined();
    expect(diag10).toBeDefined();

    // They should NOT have identical shouldIdentify — the point of pairing is to distinguish them
    const identify01 = JSON.stringify(diag01.shouldIdentify.sort());
    const identify10 = JSON.stringify(diag10.shouldIdentify.sort());
    expect(identify01).not.toBe(identify10);
  });

  it("shouldNotInfer is a non-empty array of strings", () => {
    for (const c of cases) {
      expect(Array.isArray(c.shouldNotInfer)).toBe(true);
      expect(c.shouldNotInfer.length).toBeGreaterThan(0);
      expect(typeof c.shouldNotInfer[0]).toBe("string");
    }
  });
});

// ── Mock evaluation writes correct files ───────────

describe("mock evaluation result capture", () => {
  it("test file path exists", () => {
    const path = join(__dirname, "data", "live-diagnostic-v0.2.json");
    expect(existsSync(path)).toBe(true);
  });

  it("package.json contains diagnostic scripts", async () => {
    const pkg = JSON.parse(
      readFileSync(join(rootDir, "package.json"), "utf-8")
    );
    expect(pkg.scripts["evaluate:mock"]).toContain("EVAL_REAL=0");
    expect(pkg.scripts["evaluate:diagnostic"]).toContain("EVAL_DIAGNOSTIC=1");
    expect(pkg.scripts["evaluate:live"]).toContain("EVAL_REAL=1");
  });
});

// ── Markdown generation correctness ────────────────

describe("markdown summary content", () => {
  it("contains expected header format for each case ID pattern", () => {
    const cases = JSON.parse(
      readFileSync(join(__dirname, "data", "live-diagnostic-v0.2.json"), "utf-8")
    );
    for (const c of cases) {
      expect(c.description).toBeDefined();
      expect(typeof c.description).toBe("string");
      expect(c.description.length).toBeGreaterThan(0);
    }
  });

  it("diag-01 and diag-02 have different descriptions indicating their distinction", () => {
    const cases = JSON.parse(
      readFileSync(join(__dirname, "data", "live-diagnostic-v0.2.json"), "utf-8")
    );
    const diag01 = cases.find((c) => c.id === "diag-01");
    const diag02 = cases.find((c) => c.id === "diag-02");
    expect(diag01.description).not.toBe(diag02.description);
  });
});

// ── Command safeguards ─────────────────────────────

describe("command safeguards", () => {
  it("evaluate:diagnostic sets EVAL_DIAGNOSTIC env var", async () => {
    const pkg = JSON.parse(
      readFileSync(join(rootDir, "package.json"), "utf-8")
    );
    expect(pkg.scripts["evaluate:diagnostic"]).toMatch(/EVAL_DIAGNOSTIC=1/);
  });

  it("evaluate:mock sets EVAL_REAL=0 to prevent real provider calls", async () => {
    const pkg = JSON.parse(
      readFileSync(join(rootDir, "package.json"), "utf-8")
    );
    expect(pkg.scripts["evaluate:mock"]).toMatch(/EVAL_REAL=0/);
  });

  it("evaluate:live sets EVAL_REAL=1 to enable real provider", async () => {
    const pkg = JSON.parse(
      readFileSync(join(rootDir, "package.json"), "utf-8")
    );
    expect(pkg.scripts["evaluate:live"]).toMatch(/EVAL_REAL=1/);
  });

  it("mock script does not have EVAL_DIAGNOSTIC set (avoids accidental diagnostic mode)", async () => {
    const pkg = JSON.parse(
      readFileSync(join(rootDir, "package.json"), "utf-8")
    );
    expect(pkg.scripts["evaluate:mock"]).not.toMatch(/EVAL_DIAGNOSTIC/);
  });
});

// ── Evaluator.mjs integration ──────────────────────

describe("evaluator diagnostic mode integration", () => {
  it("evaluator.mjs checks for EVAL_DIAGNOSTIC env var", async () => {
    const evaluator = readFileSync(
      join(__dirname, "..", "evaluator.mjs"),
      "utf-8"
    );
    expect(evaluator).toContain("EVAL_DIAGNOSTIC");
    expect(evaluator).toContain("useDiagnostic");
  });

  it("evaluator loads JSON array for diagnostic mode (not JSONL)", async () => {
    const evaluator = readFileSync(
      join(__dirname, "..", "evaluator.mjs"),
      "utf-8"
    );
    // Should handle .json files with JSON.parse (array format)
    expect(evaluator).toContain('path.endsWith(".json")');
  });

  it("evaluator writes to evaluation-results directory for diagnostic mode", async () => {
    const evaluator = readFileSync(
      join(__dirname, "..", "evaluator.mjs"),
      "utf-8"
    );
    expect(evaluator).toContain("evaluation-results");
  });

  it("evaluator saves per-case markdown summaries for diagnostic mode", async () => {
    const evaluator = readFileSync(
      join(__dirname, "..", "evaluator.mjs"),
      "utf-8"
    );
    expect(evaluator).toContain("-summary.md");
  });

  it("evaluator saves summary.json and manifest for diagnostic runs", async () => {
    const evaluator = readFileSync(
      join(__dirname, "..", "evaluator.mjs"),
      "utf-8"
    );
    expect(evaluator).toContain("summary.json");
    expect(evaluator).toContain("latest-manifest.json");
  });
});

// ── Live diagnostic data content verification ──────

describe("diagnostic case reasoning diversity", () => {
  const cases = JSON.parse(
    readFileSync(join(__dirname, "data", "live-diagnostic-v0.2.json"), "utf-8")
  );

  it("covers all expected primary types", () => {
    const expectedTypes = [
      "unexplained_change",
      "observed_problem",
      "contradiction",
      "decision_request",
      "reported_claim",
      "ambiguous_statement",
      "causal_claim",
    ];
    const found = new Set(cases.flatMap((c) => c.expectedPrimaryTypes));
    for (const t of expectedTypes) {
      expect(found.has(t)).toBe(true);
    }
  });

  it("diag-03 and diag-09 are distinct test targets", () => {
    const diag03 = cases.find((c) => c.id === "diag-03");
    const diag09 = cases.find((c) => c.id === "diag-09");
    expect(diag03.expectedPrimaryTypes).not.toEqual(diag09.expectedPrimaryTypes);
  });

  it("each case has a unique description", () => {
    const descs = cases.map((c) => c.description);
    const unique = new Set(descs);
    expect(unique.size).toBe(descs.length);
  });
});