confidence-engine/tests/reconstruction/semantic-clarification-weak-answer-consequence.test.js

import { describe, it, expect } from "vitest";
import { config } from "dotenv";
import path from "path";
import { fileURLToPath } from "url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
config({ path: path.resolve(__dirname, "../../.env.local") });

const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL;
const OLLAMA_MODEL = process.env.OLLAMA_MODEL;

if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) {
  throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local");
}

/**
 * Make one live Ollama chat call: take a clarification answer
 * and return what was resolved, whether the target is resolved,
 * and any remaining uncertainty about that specific target.
 */
async function callClarificationAnswerResolution(source, clarificationTarget, clarificationQuestion, userAnswer) {
  const instruction = `Use the user's clarification answer only to resolve the supplied clarification target. State the meaning now established by that answer. Mark targetResolved true only when the answer settles the target. Put any uncertainty that remains specifically about that target into remainingUncertainty; otherwise return null. Do not infer wider consequences, rewrite unrelated source meaning, recommend action, or generate another question.

Return valid JSON only in this shape:
{
  "resolvedMeaning": "short statement",
  "targetResolved": true,
  "remainingUncertainty": null
}`;

  const messages = [
    { role: "system", content: instruction.trim() },
    {
      role: "user",
      content: `Source: ${JSON.stringify(source)}

Clarification target: ${clarificationTarget}

Clarification question: ${clarificationQuestion}

User's answer: ${userAnswer}`,
    },
  ];

  const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model: OLLAMA_MODEL,
      messages,
      format: "json",
      stream: false,
    }),
  });

  if (!res.ok) {
    throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
  }

  const data = await res.json();
  const rawContent = data.message?.content ?? "";
  const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");

  return JSON.parse(cleaned.trim());
}

// ──────────────────────────────────────────────
// Fixed source (Experiment 54Z)
// ──────────────────────────────────────────────

const SOURCE = "I want the business to grow, but I don't want to take on more risk.";

// Variant A - Precise target and fixed question (from Experiment 54Y)
const VARIANT_A_TARGET = "whether avoiding additional risk is a preference/trade-off or a hard constraint";
const VARIANT_A_QUESTION = "Do you view avoiding additional risk as a hard constraint, or as a preference or trade-off?";

// Variant B - Broadened target and fixed question (from Experiment 54Y)
const VARIANT_B_TARGET = "priority between business growth and risk avoidance when they conflict";
const VARIANT_B_QUESTION = "When business growth and risk avoidance conflict, which do you prioritize?";

// Answer 1 - Priority Without Constraint Meaning
const ANSWER_1 = "Risk matters more to me.";

// Answer 2 - Conditional Trade-Off
const ANSWER_2 = "I'd normally avoid more risk, but for the right opportunity I might accept some.";

// ──────────────────────────────────────────────
// Semantic evaluation helpers
// ──────────────────────────────────────────────

/**
 * Classify whether two resolution results are materially equivalent
 * or materially different for downstream reasoning.
 */
function classifyResolutionEquivalence(resA, resB) {
  const a = resA.resolvedMeaning.toLowerCase();
  const b = resB.resolvedMeaning.toLowerCase();

  // Check hard constraint semantics in each
  const hasHardConstraint = (r) =>
    /hard.*(constraint|boundary|limit)|no.*(increase|more|additional|take on).*risk|absolut.*no.*risk|must.*(not|avoid)/i.test(r);

  // Check conditional/trade-off semantics
  const hasConditionality = (r) =>
    /\bconditionally\b|depends.*on|might?.*accept|under.*condition|depending|unless|except/i.test(r) ||
    /normally.*avoid.*but|for.*(the |some )?right.*(opportunity|case|situation)/i.test(r);

  // Check "matters more" priority semantics (no hard constraint implied)
  const hasPriorityMeaning = (r) =>
    /\bmatters.?more\b|higher.?priority|priorit.*risk|risk.*takes\s*(precedence|priority)|more\s*important\s*than/i.test(r);

  const aHasHard = hasHardConstraint(a);
  const bHasHard = hasHardConstraint(b);
  const aHasCond = hasConditionality(a);
  const bHasCond = hasConditionality(b);
  const aHasPriority = hasPriorityMeaning(a);
  const bHasPriority = hasPriorityMeaning(b);

  // If one is clearly hard constraint and the other is not - materially different
  if (aHasHard !== bHasHard) {
    return "resolutions_materially_different";
  }

  // Check conditionality divergence
  if ((aHasCond && !bHasCond) || (!aHasCond && bHasCond)) {
    return "resolutions_materially_different";
  }

  // If both resolve differently on targetResolution
  const aResolved = resA.targetResolved === true;
  const bResolved = resB.targetResolved === true;
  if (aResolved !== bResolved) {
    return "resolutions_materially_different";
  }

  // Check remaining uncertainty state divergence
  const aHasUncertainty = resA.remainingUncertainty !== null && resA.remainingUncertainty !== undefined;
  const bHasUncertainty = resB.remainingUncertainty !== null && resB.remainingUncertainty !== undefined;

  if (aHasUncertainty !== bHasUncertainty) {
    return "resolutions_materially_different";
  }

  // Both have constraint semantics
  if (aHasHard && bHasHard) {
    return "resolutions_materially_equivalent";
  }

  // Neither has hard constraint - compare priority framing consistency
  if ((aHasUncertainty && bHasUncertainty) || (!aHasUncertainty && !bHasUncertainty)) {
    const bothAboutRiskBoundary = /risk/i.test(a) && /risk/i.test(b);
    if (bothAboutRiskBoundary && aHasPriority === bHasPriority) {
      return "resolutions_materially_equivalent";
    }
  }

  // Default to manual review needed
  return "manual_review_required";
}

/**
 * Check whether the model forced stronger meaning than the answer supplies.
 */
function detectsForcedCertainty(res, userAnswer) {
  const m = res.resolvedMeaning.toLowerCase();
  const a = userAnswer.toLowerCase();

  // If the answer is vague but the resolution claims absolute/hard constraint
  if (!a.includes("constraint") && !a.includes("absolute") && !a.includes("must not")) {
    if (/must.*not.*take|no.*risk|hard.*constraint|absolute.*limit|will.*never/i.test(m)) {
      return "forced_certainty_detected";
    }
  }

  // If the answer is conditional but the resolution claims unconditional
  if (a.includes("might") || a.includes("but") || a.includes("unless") || a.includes("conditionally")) {
    if (/will.*never|must.*not|absolute|never.*accept/i.test(m) && !/normally.*but|conditional|depends/i.test(m)) {
      return "forced_certainty_detected";
    }
  }

  return null;
}

/**
 * Check whether the model erased uncertainty that should remain.
 */
function detectsErasedUncertainty(res, userAnswer) {
  const hasRemaining = res.remainingUncertainty !== null && res.remainingUncertainty !== undefined;

  // For Answer 1 (weak): if targetResolved is true and no remaining uncertainty,
  // the model may have erased uncertainty that should remain
  const answerIsWeak = !userAnswer.includes("constraint") && !userAnswer.includes("absolute");
  if (answerIsWeak && res.targetResolved === true && !hasRemaining) {
    return "potential_erasal_of_uncertainty";
  }

  return null;
}

// ──────────────────────────────────────────────
// Test suite - Experiment 54Z
// ──────────────────────────────────────────────

describe("Experiment 54Z - Clarification Broadening with Weak Answers", () => {
  const results = {
    answer1: { variantA: null, variantB: null },
    answer2: { variantA: null, variantB: null },
    timings: [],
  };

  // -- Answer 1, Variant A --

  it("Answer 1 -> Variant A (precise target)", async () => {
    const start = Date.now();
    const result = await callClarificationAnswerResolution(
      SOURCE, VARIANT_A_TARGET, VARIANT_A_QUESTION, ANSWER_1
    );
    const elapsed = Date.now() - start;
    results.timings.push({ stage: "A1-V-A", ms: elapsed });
    results.answer1.variantA = result;
  }, 120000);

  // -- Answer 1, Variant B --

  it("Answer 1 -> Variant B (broadened target)", async () => {
    const start = Date.now();
    const result = await callClarificationAnswerResolution(
      SOURCE, VARIANT_B_TARGET, VARIANT_B_QUESTION, ANSWER_1
    );
    const elapsed = Date.now() - start;
    results.timings.push({ stage: "A1-V-B", ms: elapsed });
    results.answer1.variantB = result;
  }, 120000);

  // -- Answer 2, Variant A --

  it("Answer 2 -> Variant A (precise target)", async () => {
    const start = Date.now();
    const result = await callClarificationAnswerResolution(
      SOURCE, VARIANT_A_TARGET, VARIANT_A_QUESTION, ANSWER_2
    );
    const elapsed = Date.now() - start;
    results.timings.push({ stage: "A2-V-A", ms: elapsed });
    results.answer2.variantA = result;
  }, 120000);

  // -- Answer 2, Variant B --

  it("Answer 2 -> Variant B (broadened target)", async () => {
    const start = Date.now();
    const result = await callClarificationAnswerResolution(
      SOURCE, VARIANT_B_TARGET, VARIANT_B_QUESTION, ANSWER_2
    );
    const elapsed = Date.now() - start;
    results.timings.push({ stage: "A2-V-B", ms: elapsed });
    results.answer2.variantB = result;
  }, 120000);

  // -- Aggregate Evaluation --

  it("54Z: evaluate Answer 1 pairwise comparison", () => {
    const resA = results.answer1.variantA;
    const resB = results.answer1.variantB;

    console.log("\n========== Experiment 54Z Results ==========");
    console.log("\n--- Source ---");
    console.log(SOURCE);
    console.log("\n--- Answer 1: " + ANSWER_1 + " ---");
    console.log("\nVariant A (precise):");
    console.log("  resolvedMeaning:", resA.resolvedMeaning);
    console.log("  targetResolved:", resA.targetResolved);
    console.log("  remainingUncertainty:", resA.remainingUncertainty ?? null);

    console.log("\nVariant B (broadened):");
    console.log("  resolvedMeaning:", resB.resolvedMeaning);
    console.log("  targetResolved:", resB.targetResolved);
    console.log("  remainingUncertainty:", resB.remainingUncertainty ?? null);

    const equiv = classifyResolutionEquivalence(resA, resB);
    results.answer1.classification = equiv;

    console.log("\nAnswer 1 comparison: " + equiv);

    // Forced certainty check
    const forcedA1 = detectsForcedCertainty(resA, ANSWER_1);
    const forcedB1 = detectsForcedCertainty(resB, ANSWER_1);
    if (forcedA1) console.log("Answer 1 Variant A: " + forcedA1);
    if (forcedB1) console.log("Answer 1 Variant B: " + forcedB1);

    // Erased uncertainty check
    const erasedA1 = detectsErasedUncertainty(resA, ANSWER_1);
    const erasedB1 = detectsErasedUncertainty(resB, ANSWER_1);
    if (erasedA1) console.log("Answer 1 Variant A: " + erasedA1);
    if (erasedB1) console.log("Answer 1 Variant B: " + erasedB1);

    // Answer 1 human review questions
    console.log("\n--- Manual Review Questions ---");
    console.log("Q1: Did Variant A preserve uncertainty about preference vs hard constraint?",
      !resA.targetResolved || resA.remainingUncertainty !== null ? "Yes" : "No - need review");
    console.log("Q2: Did Variant B resolve the broader priority target?",
      resB.targetResolved ? "Yes" : "No");
    console.log("Q3: Materially different resolution states for Answer 1?",
      equiv === "resolutions_materially_different" || equiv === "manual_review_required" ? "Possibly - manual review needed" : "No");

    results.answer1 = { ...results.answer1, forcedA1, forcedB1, erasedA1, erasedB1 };
  });

  it("54Z: evaluate Answer 2 pairwise comparison", () => {
    const resA = results.answer2.variantA;
    const resB = results.answer2.variantB;

    console.log("\n--- Answer 2: " + ANSWER_2 + " ---");
    console.log("\nVariant A (precise):");
    console.log("  resolvedMeaning:", resA.resolvedMeaning);
    console.log("  targetResolved:", resA.targetResolved);
    console.log("  remainingUncertainty:", resA.remainingUncertainty ?? null);

    console.log("\nVariant B (broadened):");
    console.log("  resolvedMeaning:", resB.resolvedMeaning);
    console.log("  targetResolved:", resB.targetResolved);
    console.log("  remainingUncertainty:", resB.remainingUncertainty ?? null);

    const equiv = classifyResolutionEquivalence(resA, resB);
    results.answer2.classification = equiv;

    console.log("\nAnswer 2 comparison: " + equiv);

    // Forced certainty check
    const forcedA2 = detectsForcedCertainty(resA, ANSWER_2);
    const forcedB2 = detectsForcedCertainty(resB, ANSWER_2);
    if (forcedA2) console.log("Answer 2 Variant A: " + forcedA2);
    if (forcedB2) console.log("Answer 2 Variant B: " + forcedB2);

    // Erased uncertainty check
    const erasedA2 = detectsErasedUncertainty(resA, ANSWER_2);
    const erasedB2 = detectsErasedUncertainty(resB, ANSWER_2);
    if (erasedA2) console.log("Answer 2 Variant A: " + erasedA2);
    if (erasedB2) console.log("Answer 2 Variant B: " + erasedB2);

    // Answer 2 human review questions
    console.log("\n--- Manual Review Questions ---");
    console.log("Q4: Did Variant A correctly identify risk avoidance is not absolute hard constraint?",
      !resA.targetResolved || (resA.targetResolved && resA.remainingUncertainty !== null) ? "Yes/Partially" : "No - need review");
    console.log("Q5: Did Variant B preserve conditionality?",
      /normally.*but|conditional|depends|might/i.test(resB.resolvedMeaning) ? "Yes" : "Need review");
    console.log("Q6: Materially equivalent resolution states for Answer 2?",
      equiv === "resolutions_materially_equivalent" ? "Yes" :
      equiv === "resolutions_materially_different" ? "No - they differ" :
      "Manual review needed");

    results.answer2 = { ...results.answer2, forcedA2, forcedB2, erasedA2, erasedB2 };
  });

  it("54Z: aggregate timing and summary", () => {
    const totalMs = results.timings.reduce((s, t) => s + t.ms, 0);
    const msArr = results.timings.map((t) => t.ms);

    console.log("\n--- Timing ---");
    console.log("Calls:", results.timings.length);
    console.log("Total:", totalMs + "ms");
    console.log("Average:", (totalMs / results.timings.length).toFixed(1) + "ms per call");
    console.log("Fastest:", Math.min(...msArr) + "ms");
    console.log("Slowest:", Math.max(...msArr) + "ms");

    // Summary
    const a1Diff = results.answer1.classification === "resolutions_materially_different";
    const a2Diff = results.answer2.classification === "resolutions_materially_different";
    const anyForced = [results.answer1.forcedA1, results.answer1.forcedB1, results.answer2.forcedA2, results.answer2.forcedB2].some(Boolean);
    const anyErased = [results.answer1.erasedA1, results.answer1.erasedB1, results.answer2.erasedA2, results.answer2.erasedB2].some(Boolean);

    console.log("\n--- Summary ---");
    console.log("Answer 1 materially different:", a1Diff);
    console.log("Answer 2 materially different:", a2Diff);
    console.log("Any forced certainty detected:", anyForced);
    console.log("Any uncertainty erased:", anyErased);

    results.summary = {
      totalTimeMs: totalMs,
      averageMs: (totalMs / results.timings.length).toFixed(1),
      fastestMs: Math.min(...msArr),
      slowestMs: Math.max(...msArr),
      answer1MateriallyDifferent: a1Diff,
      answer2MateriallyDifferent: a2Diff,
      anyForcedCertainty: anyForced,
      anyErasedUncertainty: anyErased,
    };

    console.log("\n========== End of Experiment 54Z ==========\n");
  });
});