confidence-engine/tests/reconstruction/semantic-clarification-null-stability.test.js

import { describe, it, expect } from "vitest";
import { config } from "dotenv";
import path from "path";
import { fileURLToPath } from "url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
config({ path: path.resolve(__dirname, "../../.env.local") });

const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL;
const OLLAMA_MODEL = process.env.OLLAMA_MODEL;

if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) {
  throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local");
}

/**
 * Make one live Ollama chat call: identify the specific user-owned
 * distinction that remains unresolved when clarification is required.
 * Uses the exact Experiment 54S instruction unchanged.
 */
async function callClarificationTarget(source, disagreement, requiresUserClarification) {
  const instruction = `Identify the specific unresolved distinction that only the user can clarify.

If clarification is required (requiresUserClarification: true), return the smallest statement of the missing user-owned meaning, preference, priority, constraint, definition, or private fact.

If clarification is not required (requiresUserClarification: false), return null.

Do not write a question. Do not add evidence needs. Do not select a preferred interpretation.

Return valid JSON only in this shape:
{
  "clarificationTarget": "short statement" | null
}`;

  const messages = [
    { role: "system", content: instruction.trim() },
    {
      role: "user",
      content: `Source: ${JSON.stringify(source)}

Disagreement:
${disagreement.map((d, i) => `${i + 1}. ${d}`).join("\n")}

requiresUserClarification: ${requiresUserClarification}`,
    },
  ];

  const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model: OLLAMA_MODEL,
      messages,
      format: "json",
      stream: false,
    }),
  });

  if (!res.ok) {
    throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
  }

  const data = await res.json();
  const rawContent = data.message?.content ?? "";
  const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");

  return JSON.parse(cleaned.trim());
}

// ──────────────────────────────────────────────
// Human reference for Case B semantic review
// ──────────────────────────────────────────────

const HUMAN_REF_CASE_B = {
  expectedDistinction:
    "whether avoiding additional risk is a preference/trade-off or a hard constraint",
  requiredConcepts: [
    "risk",
    "constraint",
    "preference",
    "trade-off",
    "avoiding",
    "additional",
    "hard",
  ],
  forbiddenConcepts: ["evidence", "investigate", "check", "look at", "data"],
};

function evaluateCaseBTarget(target) {
  if (target == null) return { classification: "null", target };
  if (typeof target !== "string" || !target.trim())
    return { classification: "null", target };

  const t = target.trim().toLowerCase();
  if (t.endsWith("?")) return { classification: "null", target };

  // Semantic check against human reference
  const reqConcepts = HUMAN_REF_CASE_B.requiredConcepts;
  const hasRequired = reqConcepts.some(
    (c) => t.includes(c.toLowerCase())
  );

  const forbiddenConcepts = HUMAN_REF_CASE_B.forbiddenConcepts;
  const hasForbidden = forbiddenConcepts.some(
    (c) => t.includes(c.toLowerCase())
  );

  if (!hasRequired || hasForbidden) {
    return { classification: "target_incorrect", target };
  }

  return { classification: "target_correct", target };
}

// ──────────────────────────────────────────────
// Fixed cases
// ──────────────────────────────────────────────

const CASE_A = {
  id: "Case A — Evidence-Resolvable / False",
  source: "Orders are arriving late and customers have started complaining.",
  disagreement: [
    "delays may be caused by insufficient staff capacity",
    "delays may be caused by unreliable supplier lead times",
  ],
  requiresUserClarification: false,
  expectedNull: true,
};

const CASE_B = {
  id: "Case B — User-Owned Ambiguity / True Control",
  source: "I want the business to grow, but I don't want to take on more risk.",
  disagreement: [
    "growth should be prioritised even if some additional risk is unavoidable",
    "avoiding additional risk is a hard constraint even if growth is slower",
  ],
  requiresUserClarification: true,
  expectedNull: false,
};

const RUNS_PER_CASE = 3;

// ──────────────────────────────────────────────
// Test suite — Case A (false → null stability)
// ──────────────────────────────────────────────

describe("Experiment 54T — Clarification Null Stability", () => {
  const results = { a: [], b: [] };
  const timings = [];

  describe("Case A — Evidence-Resolvable / False (repeated " + RUNS_PER_CASE + "×)", () => {
    for (let i = 0; i < RUNS_PER_CASE; i++) {
      it(`run ${i + 1}`, async () => {
        const start = Date.now();
        const result = await callClarificationTarget(
          CASE_A.source,
          CASE_A.disagreement,
          CASE_A.requiresUserClarification
        );
        const elapsed = Date.now() - start;
        timings.push({ caseId: CASE_A.id, run: i + 1, ms: elapsed });

        const isNull = result.clarificationTarget == null;
        results.a.push({ run: i + 1, result, isNull });

        // Structural check
        expect(result.clarificationTarget).toBeDefined();

        // Null-gating: false → null expected
        if (CASE_A.expectedNull) {
          console.log(
            `[54T Case A run ${i + 1}] clarificationTarget: ${isNull ? "null ✓" : `"${result.clarificationTarget}" ✗`}`
          );
          expect(isNull).toBe(true);
        } else {
          expect(isNull).toBe(false);
        }
      }, 120000);
    }
  });

  describe("Case B — User-Owned Ambiguity / True Control (repeated " + RUNS_PER_CASE + "×)", () => {
    for (let i = 0; i < RUNS_PER_CASE; i++) {
      it(`run ${i + 1}`, async () => {
        const start = Date.now();
        const result = await callClarificationTarget(
          CASE_B.source,
          CASE_B.disagreement,
          CASE_B.requiresUserClarification
        );
        const elapsed = Date.now() - start;
        timings.push({ caseId: CASE_B.id, run: i + 1, ms: elapsed });

        const ev = evaluateCaseBTarget(result.clarificationTarget);
        results.b.push({ run: i + 1, result, classification: ev.classification });

        // Structural check
        expect(result.clarificationTarget).toBeDefined();

        console.log(
          `[54T Case B run ${i + 1}] classification: ${ev.classification} | target: "${result.clarificationTarget ?? "null"}"`
        );

        // true → non-null expected
        if (CASE_B.expectedNull) {
          expect(ev.classification).not.toBe("null");
        } else {
          // This case expects a non-null correct target
          expect(ev.classification).toBe("target_correct");
        }
      }, 120000);
    }
  });

  // ──────────────────────────────────────────────
  // Aggregate summary
  // ──────────────────────────────────────────────

  it("54T aggregate results", () => {
    const aNull = results.a.filter((r) => r.isNull).length;
    const aNonNull = results.a.filter((r) => !r.isNull).length;

    const bCorrect = results.b.filter((r) => r.classification === "target_correct").length;
    const bIncorrect = results.b.filter((r) => r.classification === "target_incorrect").length;
    const bNull = results.b.filter((r) => r.classification === "null").length;

    const totalMs = timings.reduce((s, t) => s + t.ms, 0);
    const avgMs = totalMs / timings.length;
    const fastMs = Math.min(...timings.map((t) => t.ms));
    const slowMs = Math.max(...timings.map((t) => t.ms));

    console.log("\n=== Experiment 54T Aggregate Results ===");
    console.log(`\n--- Case A (false → null) ---`);
    for (const r of results.a) {
      console.log(
        `Run ${r.run}: ${r.isNull ? "null" : `"${r.result.clarificationTarget}"`}`
      );
    }
    console.log(`Null: ${aNull}/${results.a.length}`);
    console.log(`Non-null: ${aNonNull}/${results.a.length}`);

    if (aNonNull > 0) {
      console.log("\nInvented clarification targets:");
      results
        .filter((r) => !r.isNull)
        .forEach((r) => console.log(`  - "${r.result.clarificationTarget}"`));
    }

    console.log(`\n--- Case B (true → target) ---`);
    for (const r of results.b) {
      console.log(
        `Run ${r.run}: ${r.classification} | "${r.result ?? "null"}"`
      );
    }
    console.log(`Correct: ${bCorrect}/${results.b.length}`);
    console.log(`Incorrect: ${bIncorrect}/${results.b.length}`);
    console.log(`Null: ${bNull}/${results.b.length}`);

    console.log(`\n--- Timing ---`);
    console.log(`Total live calls: ${timings.length}`);
    console.log(`Total time: ${totalMs}ms (${(totalMs / 1000).toFixed(1)}s)`);
    console.log(`Average: ${avgMs.toFixed(1)}ms per call`);
    console.log(`Fastest: ${fastMs}ms`);
    console.log(`Slowest: ${slowMs}ms`);

    // Summary assertions
    expect(timings.length).toBe(6);
    expect(aNull + aNonNull).toBe(3);
    expect(bCorrect + bIncorrect + bNull).toBe(3);
  });
});