import { describe, it, expect } from "vitest";
import { config } from "dotenv";
import path from "path";
import { fileURLToPath } from "url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
config({ path: path.resolve(__dirname, "../../.env.local") });

const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL;
const OLLAMA_MODEL = process.env.OLLAMA_MODEL;

if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) {
  throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local");
}

/**
 * Make one live Ollama chat call: decide whether a disagreement requires
 * user clarification or can be resolved through evidence.
 */
async function callDisagreementResolutionSource(source, disagreement, evidenceNeeded) {
  const instruction = `Decide whether resolving the stated disagreement requires additional meaning, preference, intent, or factual information that only the user can provide.

Return true when evidence alone cannot settle the disagreement because the missing distinction belongs to the user's intended meaning, priority, constraint, or private knowledge. Return false when the disagreement can be investigated using external, operational, or observable evidence without asking the user to define what they mean.

Return valid JSON only in this shape:
{
  "requiresUserClarification": true | false,
  "reason": "one short sentence"
}

Do not generate a question. Do not choose which interpretation is correct.`;

  const messages = [
    { role: "system", content: instruction.trim() },
    {
      role: "user",
      content: `Source: ${JSON.stringify(source)}

Disagreement:
${disagreement.map((d, i) => `${i + 1}. ${d}`).join("\n")}

Evidence needed:
${evidenceNeeded.map((e, i) => `${i + 1}. ${e}`).join("\n")}`,
    },
  ];

  const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model: OLLAMA_MODEL,
      messages,
      format: "json",
      stream: false,
    }),
  });

  if (!res.ok) {
    throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
  }

  const data = await res.json();
  const rawContent = data.message?.content ?? "";
  const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");

  return JSON.parse(cleaned.trim());
}

// ──────────────────────────────────────────────
// Fixed human-reference ground truth (pre-written)
// ──────────────────────────────────────────────

const CASES = [
  {
    id: "Case 1 - Competing Causes, Evidence Can Resolve",
    source: "Orders are arriving late and customers have started complaining.",
    disagreement: [
      "delays may be caused by insufficient staff capacity",
      "delays may be caused by unreliable supplier lead times",
    ],
    evidenceNeeded: [
      "staffing levels and workload",
      "processing throughput",
      "supplier lead-time history",
      "supplier delivery reliability",
    ],
    reference: {
      requiresUserClarification: false,
      reason_semantic: "The user's statement is clear enough. Operational evidence can distinguish the competing causes.",
    },
  },
  {
    id: "Case 2 - User Priority Is Ambiguous",
    source: "I want the business to grow, but I don't want to take on more risk.",
    disagreement: [
      "growth should be prioritised even if some additional risk is unavoidable",
      "avoiding additional risk is a hard constraint even if growth is slower",
    ],
    evidenceNeeded: [
      "possible growth opportunities",
      "risk exposure of each option",
    ],
    reference: {
      requiresUserClarification: true,
      reason_semantic: "External evidence can describe growth and risk, but it cannot determine whether the constraint is a preference or an absolute boundary. That meaning belongs to the user.",
    },
  },
  {
    id: "Case 3 - Same Meaning, No Clarification Need",
    source: "Revenue is down. I think pricing may be part of the problem, but I am not sure.",
    disagreement: [
      "no substantive disagreement; two interpretations express the same uncertainty in different words",
    ],
    evidenceNeeded: [
      "pricing history",
      "sales/conversion response",
      "other plausible revenue drivers",
    ],
    reference: {
      requiresUserClarification: false,
      reason_semantic: "There is no material interpretation disagreement requiring clarification. Evidence can investigate the underlying uncertainty.",
    },
  },
];

// ──────────────────────────────────────────────
// Evaluation helpers
// ──────────────────────────────────────────────

function evaluateResolutionSource(modelResult, reference) {
  const result = modelResult;
  const issues = [];
  const notes = [];

  // Structural checks
  if (typeof result.requiresUserClarification !== "boolean") {
    issues.push("requiresUserClarification must be a boolean");
  }
  if (!result.reason || typeof result.reason !== "string") {
    issues.push("missing or invalid reason field");
  }

  // Boolean match against fixed human reference
  if (result.requiresUserClarification !== reference.requiresUserClarification) {
    issues.push(
      `boolean mismatch: expected ${reference.requiresUserClarification}, got ${result.requiresUserClarification}`
    );
  }

  return {
    resolution_source_correct: issues.filter((i) =>
      i.includes("boolean mismatch")
    ).length === 0,
    resolution_source_failed: issues.some((i) =>
      i.includes("boolean mismatch")
    ),
    boolean_match: result.requiresUserClarification === reference.requiresUserClarification,
    issues,
    notes,
    raw: result,
  };
}

function evaluateReasonSemantics(modelResult, caseRef) {
  const result = modelResult;
  let issues = [];

  if (!result.reason || typeof result.reason !== "string") {
    return { valid_reason: false, issues: ["no reason provided"] };
  }

  const reasonLower = result.reason.toLowerCase();

  if (caseRef.requiresUserClarification === false) {
    // Should reference evidence / operational investigation, not user meaning
    const hasEvidenceKeywords = [
      "evidence", "operational", "investigat", "distinguish",
      "data can", "external", "observable", "record", "metric",
      "check", "review", "gather", "collect"
    ].some(k => reasonLower.includes(k));

    const hasUserMeaningKeywords = [
      "user must", "ask the user", "clarification from", "need to ask",
      "requires clarification", "unclear what the user", "ambiguous user"
    ].some(k => reasonLower.includes(k));

    // If it references evidence investigation, that supports the classification
    if (hasEvidenceKeywords) {
      issues.push("reason references evidence-based resolution for a false case - check for contradiction");
    }
  }

  if (caseRef.requiresUserClarification === true) {
    const hasUserPriority = [
      "user's", "user mean", "user intent", "user priorit",
      "preference", "constraint", "boundary", "ambiguous",
      "clarification needed"
    ].some(k => reasonLower.includes(k));

    if (!hasUserPriority) {
      issues.push("reason does not reference user-owned meaning for a true case");
    }
  }

  return {
    valid_reason: issues.length === 0,
    issues,
  };
}

function checkNoInvariantViolations(modelResult) {
  const allText = JSON.stringify(modelResult).toLowerCase();
  const violations = [];

  // No question generated
  if (/should i|do you|could you|would you|are you/.test(allText)) {
    violations.push("potential question language detected in output");
  }

  // No interpretation chosen
  if (/\b(winner|preferred|correct|right|better|should go with)\b/i.test(
      modelResult.reason || ""
  )) {
    violations.push("potential interpretation selection in reason field");
  }

  return violations;
}

// ──────────────────────────────────────────────
// Test suite
// ──────────────────────────────────────────────

describe("Experiment 54R - Disagreement Resolution Source: Clarification vs Evidence", () => {
  const results = [];
  const timings = [];

  for (const c of CASES) {
    it(c.id, async () => {
      const start = Date.now();
      const result = await callDisagreementResolutionSource(
        c.source,
        c.disagreement,
        c.evidenceNeeded
      );
      const elapsed = Date.now() - start;
      timings.push({ caseId: c.id, ms: elapsed });

      const resolutionEval = evaluateResolutionSource(result, c.reference);
      const reasonEval = evaluateReasonSemantics(result, c.reference);
      const invariantViolations = checkNoInvariantViolations(result);

      results.push({
        case: c,
        modelResult: result,
        resolution: resolutionEval,
        reason: reasonEval,
        invariantViolations,
        timingMs: elapsed,
      });

      // Structural assertions
      expect(result.requiresUserClarification).toBeDefined();
      expect(typeof result.requiresUserClarification).toBe("boolean");
      expect(result.reason).toBeDefined();
      expect(typeof result.reason).toBe("string");

      // No question generation invariant
      const allText = JSON.stringify(result).toLowerCase();
      expect(allText).not.toMatch(/should i|do you|could you|would you/);

      // Boolean must match fixed human reference
      expect(result.requiresUserClarification).toBe(c.reference.requiresUserClarification);
    }, 120000);
  }

  it("Experiment 54R: aggregate results", () => {
    const resolutionCorrect = results.filter(
      (r) => r.resolution.resolution_source_correct
    ).length;
    const resolutionFailed = results.filter(
      (r) => r.resolution.resolution_source_failed
    ).length;

    // Report findings to console for manual review
    console.log("\n=== Experiment 54R Results ===");
    for (const r of results) {
      console.log(`\n--- ${r.case.id} ---`);
      console.log("Boolean:", r.modelResult.requiresUserClarification);
      console.log("Reason:", r.modelResult.reason);
      console.log(
        "Resolution source correct:",
        r.resolution.resolution_source_correct,
        r.resolution.issues
      );
      console.log("Valid reason:", r.reason.valid_reason, r.reason.issues);
      console.log(
        "Invariant violations:",
        r.invariantViolations.length > 0 ? r.invariantViolations : "none"
      );
    }
    console.log(`\nResolution-source-correct: ${resolutionCorrect}/${results.length}`);
    console.log(`Resolution-source-failed: ${resolutionFailed}/${results.length}`);

    const totalMs = timings.reduce((s, t) => s + t.ms, 0);
    console.log(`Total time: ${totalMs}ms`);
    console.log(`Average time: ${(totalMs / timings.length).toFixed(1)}ms per call`);
    console.log(`Fastest: ${Math.min(...timings.map((t) => t.ms))}ms`);
    console.log(`Slowest: ${Math.max(...timings.map((t) => t.ms))}ms`);

    // Case 1 must be false (evidence can resolve)
    const case1 = results.find((r) => r.case.id.includes("Case 1"));
    expect(case1.modelResult.requiresUserClarification).toBe(false);

    // Case 2 must be true (user priority ambiguous)
    const case2 = results.find((r) => r.case.id.includes("Case 2"));
    expect(case2.modelResult.requiresUserClarification).toBe(true);

    // Case 3 must be false (no material disagreement)
    const case3 = results.find((r) => r.case.id.includes("Case 3"));
    expect(case3.modelResult.requiresUserClarification).toBe(false);

    // No invariant violations in any result
    for (const r of results) {
      expect(r.invariantViolations.length).toBe(0);
    }
  });
});