confidence-engine/tests/reconstruction/semantic-regression-e-evidence-vs-clarification.test.js

import { describe, it, expect } from "vitest";
import { config } from "dotenv";
import path from "path";
import { fileURLToPath } from "url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
config({ path: path.resolve(__dirname, "../../.env.local") });

const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL;
const OLLAMA_MODEL = process.env.OLLAMA_MODEL;

if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) {
  throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local");
}

// ── Regression E — Evidence-resolvable disagreement (exact case) ──
// Source: docs/reasoning-refinement-requirements.md § "Regression E"
//
// Source:        Delivery delay concern.
// Competing:     "Staff capacity may be the issue" / "Supplier lead times are likely responsible."
// Expected:      Two distinct hypotheses about causation; evidence can resolve which is correct.
// Must not:      Generate a user-facing clarification question when evidence sources can distinguish the hypotheses.

const REGRESSION_E_CASE = {
  source: "Delivery delay concern",
  competingCauses: [
    "Staff capacity may be the issue",
    "Supplier lead times are likely responsible",
  ],
};

// Pre-written human reference (derived from recorded Regression E requirement)
// Written BEFORE inspecting the model output.
const HUMAN_REFERENCE = {
  correctUncertaintyType: "evidence_needed",
  rationale: `The unresolved disagreement can be reduced by obtaining relevant evidence. It must not be treated as missing user-owned meaning merely because the engine does not yet know which interpretation is correct. A correct result should preserve the difference between evidence needed to determine what is true, and clarification needed because only the user can establish what they mean, prefer, intend, define, or constrain.`,
};

// Experiment prompt — fixed case presented with clear output contract
const EXPERIMENT_PROMPT = `Context: The user has a delivery delay concern and has offered two competing causal explanations without identifying which one is correct.

Competing hypotheses provided by the user:
- "Staff capacity may be the issue"
- "Supplier lead times are likely responsible."

These are both plausible causes of the same observed problem (delivery delay). An external party could investigate to determine which is actually true — for example, by checking current staffing levels and supplier lead time data.

You must classify the type of uncertainty present in this situation. Distinguish between:
- evidence_needed: The disagreement or gap can be reduced by obtaining relevant evidence from the world (e.g., checking facts, gathering data, consulting sources). Only the engine knows what evidence to seek, not the user.
- user_clarification_needed: Only the user can establish their own meaning, preference, intent, definition, or constraint. No external evidence can resolve it because it is about what the user means, not about what is objectively true.

Return valid JSON only in this shape:
{
  "uncertaintyType": "evidence_needed | user_clarification_needed | unresolved",
  "reason": "short explanation",
  "evidenceNeeded": "string or null"
}

Do not include a userClarificationNeeded field.`;

async function call() {
  const start = Date.now();

  const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model: OLLAMA_MODEL,
      messages: [
        { role: "system", content: "You are an analysis engine. Return only valid JSON matching the contract requested." },
        { role: "user", content: EXPERIMENT_PROMPT },
      ],
      format: "json",
      stream: false,
    }),
  });

  const elapsed = Date.now() - start;

  if (!res.ok) {
    return { error: `Ollama API error: ${res.status} ${res.statusText}`, durationMs: elapsed };
  }

  const data = await res.json();
  const rawContent = data.message?.content ?? "";
  const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");

  try {
    return { parsed: JSON.parse(cleaned.trim()), durationMs: elapsed };
  } catch (e) {
    return { parseError: e.message, rawContent: cleaned, durationMs: elapsed };
  }
}

describe("Experiment 56K — Evidence versus clarification boundary", () => {
  it(
    "Regression E: one live call — evidence vs clarification",
    async () => {
      const result = await call();

      // Structural assertions only
      expect(result).not.toHaveProperty("error");
      expect(result).not.toHaveProperty("parseError");
      expect(result.parsed).toBeDefined();
      expect(typeof result.parsed.uncertaintyType).toBe("string");
      expect(["evidence_needed", "user_clarification_needed", "unresolved"]).toContain(result.parsed.uncertaintyType);
      expect(typeof result.parsed.reason).toBe("string");
      expect(result.parsed.reason.length).toBeGreaterThan(0);

      // Raw evidence for human review
      const raw = JSON.stringify(result.parsed, null, 2);
      console.log("\n========== Experiment 56K Results ==========");
      console.log(`\nModel: ${OLLAMA_MODEL}`);
      console.log(`Base URL: ${OLLAMA_BASE_URL}`);
      console.log(`Duration: ${result.durationMs}ms`);

      console.log(`\n--- Regression E fixed case ---`);
      console.log("Source:", REGRESSION_E_CASE.source);
      console.log("Competing causes:");
      REGRESSION_E_CASE.competingCauses.forEach((c, i) => console.log(`  [${i + 1}] ${c}`));

      console.log(`\n--- Pre-written human reference ---`);
      console.log("Correct type:", HUMAN_REFERENCE.correctUncertaintyType);
      console.log("Rationale:", HUMAN_REFERENCE.rationale);

      console.log(`\n--- Raw structured response ---`);
      console.log(raw);

      // Automated classification summary
      const ut = result.parsed.uncertaintyType;
      const modelChoice = ut === "evidence_needed" ? "EVIDENCE NEEDED" :
                          ut === "user_clarification_needed" ? "USER Clarification Needed" :
                          "UNRESOLVED";

      console.log(`\n--- Model output ---`);
      console.log("uncertaintyType:", ut);
      console.log("reason:", result.parsed.reason);
      if (result.parsed.evidenceNeeded !== undefined) {
        console.log("evidenceNeeded:", result.parsed.evidenceNeeded);
      }

      // Automated checks (semantic review is authoritative)
      expect(ut).not.toBe("");
      expect(result.durationMs).toBeGreaterThan(0);

      const humanClass = ut === "evidence_needed" ? "PASS" :
                         ut === "user_clarification_needed" ? "FAIL" :
                         "UNRESOLVED";

      console.log(`\n--- Automated classification ---`);
      console.log("Human classification:", humanClass);
      console.log("Rationale: model chose", modelChoice, HUMAN_REFERENCE.correctUncertaintyType === ut ? "(matches expectation)" : "(deviates from expectation)");

      console.log("\n========== End of Experiment 56K ==========\n");

      return { result: result.parsed, durationMs: result.durationMs };
    },
    120000
  );
});