confidence-engine/tests/reconstruction/semantic-clarification-uncertainty-preservation.test.js

import { describe, it, expect } from "vitest";
import { config } from "dotenv";
import path from "path";
import { fileURLToPath } from "url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
config({ path: path.resolve(__dirname, "../../.env.local") });

const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL;
const OLLAMA_MODEL = process.env.OLLAMA_MODEL;

if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) {
  throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local");
}

/**
 * Make one live Ollama chat call: take a clarification answer
 * and return what was resolved, whether the target is resolved,
 * and any remaining uncertainty about that specific target.
 */
async function callClarificationAnswerResolution(source, clarificationTarget, clarificationQuestion, userAnswer) {
  const instruction = `Use the user's clarification answer only to resolve the supplied clarification target. State the meaning now established by that answer. Mark targetResolved true only when the answer settles the target. Put any uncertainty that remains specifically about that target into remainingUncertainty; otherwise return null. Do not infer wider consequences, rewrite unrelated source meaning, recommend action, or generate another question.

Return valid JSON only in this shape:
{
  "resolvedMeaning": "short statement",
  "targetResolved": true,
  "remainingUncertainty": null
}`;

  const messages = [
    { role: "system", content: instruction.trim() },
    {
      role: "user",
      content: `Source: ${JSON.stringify(source)}

Clarification target: ${clarificationTarget}

Clarification question: ${clarificationQuestion}

User's answer: ${userAnswer}`,
    },
  ];

  const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model: OLLAMA_MODEL,
      messages,
      format: "json",
      stream: false,
    }),
  });

  if (!res.ok) {
    throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
  }

  const data = await res.json();
  const rawContent = data.message?.content ?? "";
  const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");

  return JSON.parse(cleaned.trim());
}

// ──────────────────────────────────────────────
// Experiment 55A — Fixed source, target, question
// ──────────────────────────────────────────────

const SOURCE = "I want the business to grow, but I don't want to take on more risk.";
const CLARIFICATION_TARGET = "whether avoiding additional risk is a preference/trade-off or a hard constraint";
const CLARIFICATION_QUESTION = "Do you view avoiding additional risk as a hard constraint, or as a preference or trade-off?";

// ──────────────────────────────────────────────
// Four fixed cases
// ──────────────────────────────────────────────

const CASES = [
  {
    id: "Case 1 — Fully Explicit Hard Constraint",
    userAnswer: "It's a hard constraint. I don't want any increase in risk.",
    humanReference: {
      resolvedMeaning: "avoiding additional risk is a hard constraint",
      targetResolved: true,
      remainingUncertainty: null,
    },
    purpose: "Positive control — explicit hard constraint should resolve cleanly.",
  },
  {
    id: "Case 2 — Weak Priority Statement",
    userAnswer: "Risk matters more to me.",
    humanReference: {
      resolvedMeaning: null, // do not accept a resolved meaning that infers risk is not a hard constraint
      targetResolved: false,
      remainingUncertainty: "whether avoiding additional risk is a preference/trade-off or a hard constraint remains unclear",
    },
    purpose: "Weak answer should preserve uncertainty rather than over-resolve.",
  },
  {
    id: "Case 3 — Conditional Trade-Off",
    userAnswer: "I'd normally avoid more risk, but for the right opportunity I might accept some.",
    humanReference: {
      resolvedMeaning: null, // should preserve conditional qualification, not flatten to "simply a preference"
      targetResolved: true,
      remainingUncertainty: null,
    },
    purpose: "Resolved answer should preserve conditionality without flattening.",
  },
  {
    id: "Case 4 — Non-Answer / Insufficient Clarification",
    userAnswer: "I'm not really sure.",
    humanReference: {
      resolvedMeaning: null, // should not invent a position
      targetResolved: false,
      remainingUncertainty: "whether avoiding additional risk is a preference/trade-off or a hard constraint remains unresolved",
    },
    purpose: "Non-answer should remain unresolved without inventing meaning.",
  },
];

// ──────────────────────────────────────────────
// Semantic evaluation
// ──────────────────────────────────────────────

/**
 * Classify the model's output for each case.
 * Returns { classification, reasoning } where classification is one of:
 *   resolution_correct, uncertainty_preserved, uncertainty_overresolved, resolution_failed
 */
function classifyCase(caseNum, modelResult) {
  const { resolvedMeaning, targetResolved, remainingUncertainty } = modelResult;

  // Structural checks first
  if (typeof resolvedMeaning !== "string" || !resolvedMeaning.trim()) {
    return { classification: "resolution_failed", reasoning: "missing or empty resolvedMeaning" };
  }
  if (typeof targetResolved !== "boolean") {
    return { classification: "resolution_failed", reasoning: `targetResolved is ${typeof targetResolved}, expected boolean` };
  }

  const meaning = resolvedMeaning.toLowerCase();

  // Check whether the model produced stronger meaning than the user supplied
  const hasForcedStronger = () => {
    // Force hard constraint where user didn't supply it
    if (caseNum !== 1 && /hard.*constraint|no.*increase|must.*not|absolute.*limit|will.*never/i.test(meaning)) {
      return true;
    }
    // Flatten conditionality
    if (caseNum === 3) {
      if (/simply a preference|risk is just|i would always avoid/i.test(meaning) && !/but|condition|might|would normally/i.test(meaning)) {
        return true;
      }
    }
    return false;
  };

  // Case-specific classification
  switch (caseNum) {
    case 1: {
      // Expected: targetResolved=true, no remaining uncertainty, meaning says hard constraint
      if (targetResolved !== true) {
        return { classification: "resolution_failed", reasoning: `Expected targetResolved=true, got ${targetResolved}` };
      }
      if (remainingUncertainty !== null && remainingUncertainty !== undefined) {
        return { classification: "resolution_correct", reasoning: `Correctly resolved but retained uncertainty "${remainingUncertainty}" when null was expected` };
      }
      if (/hard.*(constraint|boundary|limit)|no.*(increase|more).*risk|must.*not.*take/i.test(meaning)) {
        return { classification: "resolution_correct", reasoning: "Correctly resolved explicit hard constraint with appropriate strength." };
      }
      return { classification: "uncertainty_overresolved", reasoning: `Meaning does not clearly express hard constraint: "${resolvedMeaning}"` };
    }

    case 2: {
      // Expected: targetResolved=false, remainingUncertainty preserves the distinction
      if (targetResolved === true) {
        return { classification: "uncertainty_overresolved", reasoning: `Over-resolved weak answer: targetResolved=true when it should be false. Meaning: "${resolvedMeaning}"` };
      }
      if (!/risk.*preference|trade.?off|constraint|preference.*(vs|or|versus)|uncertain.*risk/i.test(meaning) && meaning.includes("not")) {
        // If the model explicitly says "not a hard constraint" that's over-resolving
        if (/not.*hard.*constraint|clearly.*not.*constraint|definitely.*not/i.test(meaning)) {
          return { classification: "uncertainty_overresolved", reasoning: `Model inferred "risk is not a hard constraint" from weak answer "${resolvedMeaning}"` };
        }
      }
      if (remainingUncertainty === null || remainingUncertainty === undefined) {
        return { classification: "uncertainty_overresolved", reasoning: "targetResolved=false but no remainingUncertainty provided — uncertainty erased." };
      }
      // Check that remainingUncertainty preserves the preference vs constraint distinction
      if (/preference|trade.?off|constraint|uncertain.*distinction|preference.*(vs|or)/i.test(remainingUncertainty)) {
        return { classification: "uncertainty_preserved", reasoning: `Weak answer correctly left unresolved with appropriate remaining uncertainty.` };
      }
      return { classification: "uncertainty_preserved", reasoning: `targetResolved=false with remainingUncertainty provided (semantic match unclear but structurally correct).` };
    }

    case 3: {
      // Expected: targetResolved=true, meaning preserves conditionality
      if (targetResolved !== true) {
        return { classification: "resolution_failed", reasoning: `Expected targetResolved=true, got ${targetResolved}` };
      }
      if (/but|condition|might|would normally|conditional/i.test(meaning)) {
        return { classification: "resolution_correct", reasoning: "Correctly resolved conditional trade-off while preserving its conditional qualification." };
      }
      // Check for over-resolution (flattening)
      if (/simply.*preference|risk is just|i would always|definitely not.*hard constraint/i.test(meaning)) {
        return { classification: "uncertainty_overresolved", reasoning: `Flattened conditional answer to flat meaning: "${resolvedMeaning}"` };
      }
      return { classification: "resolution_correct", reasoning: `Resolved with meaning "${resolvedMeaning}" — conditionality may or may not be explicit.` };
    }

    case 4: {
      // Expected: targetResolved=false, no invented position, remainingUncertainty states unresolved
      if (targetResolved === true) {
        return { classification: "resolution_failed", reasoning: `Over-resolved non-answer: targetResolved=true when it should be false.` };
      }
      // Check for invented meaning
      const hasInventedPosition = /will.*avoid|would.*never|always avoid/i.test(meaning);
      if (hasInventedPosition) {
        return { classification: "uncertainty_overresolved", reasoning: `Model invented a position from non-answer: "${resolvedMeaning}"` };
      }
      // Check that the distinction is stated as unresolved in remainingUncertainty
      if (remainingUncertainty === null || remainingUncertainty === undefined) {
        return { classification: "uncertainty_preserved", reasoning: `Correctly left target unresolved with no remaining uncertainty — the distinction is genuinely absent.` };
      }
      if (/unresolved|remains unclear|still unknown|preference.*constraint/i.test(remainingUncertainty)) {
        return { classification: "uncertainty_preserved", reasoning: "Non-answer correctly left unresolved without inventing meaning." };
      }
      return { classification: "uncertainty_preserved", reasoning: `targetResolved=false with remainingUncertainty provided (structurally correct).` };
    }

    default:
      return { classification: "resolution_failed", reasoning: "Unrecognized case number" };
  }
}

// ──────────────────────────────────────────────
// Test suite
// ──────────────────────────────────────────────

describe("Experiment 55A - Clarification Uncertainty Preservation", () => {
  const results = [];
  const timings = [];

  for (let i = 0; i < CASES.length; i++) {
    const c = CASES[i];
    const caseNum = i + 1;

    it(`${c.id}`, async () => {
      const start = Date.now();
      const result = await callClarificationAnswerResolution(
        SOURCE, CLARIFICATION_TARGET, CLARIFICATION_QUESTION, c.userAnswer
      );
      const elapsed = Date.now() - start;
      timings.push({ caseId: c.id, ms: elapsed });

      const ev = classifyCase(caseNum, result);

      results.push({
        caseNumber: caseNum,
        case: c,
        modelResult: result,
        classification: ev.classification,
        reasoning: ev.reasoning,
        timingMs: elapsed,
      });

      // Structural assertions for all cases
      expect(result.resolvedMeaning).toBeDefined();
      expect(typeof result.resolvedMeaning).toBe("string");
      expect(result.resolvedMeaning.trim().length).toBeGreaterThan(0);
      expect(result.targetResolved).toBeDefined();
      expect(typeof result.targetResolved).toBe("boolean");

      // Case-specific semantic assertions from human reference
      if (caseNum === 1) {
        expect(result.targetResolved).toBe(true);
      } else if (caseNum === 2) {
        expect(result.targetResolved).toBe(false);
      } else if (caseNum === 3) {
        expect(result.targetResolved).toBe(true);
      } else if (caseNum === 4) {
        expect(result.targetResolved).toBe(false);
      }
    }, 120000);
  }

  it("55A: aggregate results and analysis", () => {
    const classificationCounts = {
      resolution_correct: 0,
      uncertainty_preserved: 0,
      uncertainty_overresolved: 0,
      resolution_failed: 0,
    };

    for (const r of results) {
      classificationCounts[r.classification]++;
    }

    const totalMs = timings.reduce((s, t) => s + t.ms, 0);
    const msArr = timings.map((t) => t.ms);

    console.log("\n========== Experiment 55A Results ==========");
    console.log(`\nSource: ${SOURCE}`);
    console.log(`Target: ${CLARIFICATION_TARGET}`);
    console.log(`Question: ${CLARIFICATION_QUESTION}`);

    for (const r of results) {
      console.log(`\n--- Case ${r.caseNumber}: ${r.case.purpose} ---`);
      console.log("Answer:", `"${r.case.userAnswer}"`);
      console.log("resolvedMeaning:", r.modelResult.resolvedMeaning);
      console.log("targetResolved:", r.modelResult.targetResolved);
      console.log("remainingUncertainty:", r.modelResult.remainingUncertainty ?? "null");
      console.log("Classification:", r.classification);
      console.log("Reasoning:", r.reasoning);
    }

    console.log("\n--- Classification Counts ---");
    Object.entries(classificationCounts).forEach(([k, v]) => {
      console.log(`${k}: ${v}`);
    });

    console.log("\n--- Timing ---");
    console.log("Calls:", timings.length);
    console.log("Total:", totalMs + "ms");
    console.log("Average:", (totalMs / timings.length).toFixed(1) + "ms per call");
    console.log("Fastest:", Math.min(...msArr) + "ms");
    console.log("Slowest:", Math.max(...msArr) + "ms");

    // Summary questions
    const q2_preserved = results[1].classification === "uncertainty_preserved";
    const q3_conditionality = /resolution_correct/i.test(results[2].reasoning);
    const q4_unresolved = results[3].classification !== "resolution_failed" && !/invented.*position/i.test(results[3].reasoning.toLowerCase());

    console.log("\n--- Key Questions ---");
    console.log("Q1: Case 1 correctly resolved explicit hard constraint?", results[0].classification === "resolution_correct" || results[0].classification === "uncertainty_overresolved" ? "Yes (resolved)" : "No");
    console.log("Q2: Case 2 preserved uncertainty rather than over-resolving?", q2_preserved ? "Yes" : "No — over-resolved or failed");
    console.log("Q3: Case 3 preserved conditional trade-off?", q3_conditionality ? "Yes" : "Need review");
    console.log("Q4: Case 4 remained unresolved without inventing meaning?", q4_unresolved ? "Yes" : "No");

    const anyOverresolving = classificationCounts.uncertainty_overresolved > 0;
    const anyFailed = classificationCounts.resolution_failed > 0;

    console.log("\n--- Summary ---");
    console.log("Any over-resolution of weak answers:", anyOverresolving);
    console.log("Any resolution failures:", anyFailed);
    console.log("Conclusion: The answer-resolution step", anyOverresolving ? "does appear to over-resolve some weak answers" : "appears honest about uncertainty levels");

    results.summary = {
      classificationCounts,
      totalTimeMs: totalMs,
      averageMs: (totalMs / timings.length).toFixed(1),
      fastestMs: Math.min(...msArr),
      slowestMs: Math.max(...msArr),
      anyOverresolution: anyOverresolving,
    };

    console.log("\n========== End of Experiment 55A ==========\n");
  });
});