confidence-engine/tests/reconstruction/semantic-answer-meaning-vs-resolution.test.js

import { describe, it, expect } from "vitest";
import { config } from "dotenv";
import path from "path";
import { fileURLToPath } from "url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
config({ path: path.resolve(__dirname, "../../.env.local") });

const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL;
const OLLAMA_MODEL = process.env.OLLAMA_MODEL;

if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) {
  throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local");
}

// ──────────────────────────────────────────────
// Mode A — Meaning Only
// ──────────────────────────────────────────────

async function callMeaningOnly(source, clarificationQuestion, userAnswer) {
  const instruction = `State only what the user's answer establishes in relation to the clarification question. Preserve uncertainty, conditionality, and qualification exactly as supplied. Do not decide whether the clarification target is resolved. Do not infer what the user did not say. Do not recommend action or generate another question.

Return valid JSON only in this shape:
{
  "answerMeaning": "short statement"
}`;

  const messages = [
    { role: "system", content: instruction.trim() },
    {
      role: "user",
      content: `Source: ${JSON.stringify(source)}

Clarification question: ${clarificationQuestion}

User's answer: ${userAnswer}`,
    },
  ];

  const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model: OLLAMA_MODEL,
      messages,
      format: "json",
      stream: false,
    }),
  });

  if (!res.ok) {
    throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
  }

  const data = await res.json();
  const rawContent = data.message?.content ?? "";
  const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");

  return JSON.parse(cleaned.trim());
}

// ──────────────────────────────────────────────
// Mode B — Resolution (unchanged from Experiment 54V / 55A)
// ──────────────────────────────────────────────

async function callClarificationAnswerResolution(source, clarificationTarget, clarificationQuestion, userAnswer) {
  const instruction = `Use the user's clarification answer only to resolve the supplied clarification target. State the meaning now established by that answer. Mark targetResolved true only when the answer settles the target. Put any uncertainty that remains specifically about that target into remainingUncertainty; otherwise return null. Do not infer wider consequences, rewrite unrelated source meaning, recommend action, or generate another question.

Return valid JSON only in this shape:
{
  "resolvedMeaning": "short statement",
  "targetResolved": true,
  "remainingUncertainty": null
}`;

  const messages = [
    { role: "system", content: instruction.trim() },
    {
      role: "user",
      content: `Source: ${JSON.stringify(source)}

Clarification target: ${clarificationTarget}

Clarification question: ${clarificationQuestion}

User's answer: ${userAnswer}`,
    },
  ];

  const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model: OLLAMA_MODEL,
      messages,
      format: "json",
      stream: false,
    }),
  });

  if (!res.ok) {
    throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
  }

  const data = await res.json();
  const rawContent = data.message?.content ?? "";
  const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");

  return JSON.parse(cleaned.trim());
}

// ──────────────────────────────────────────────
// Fixed source, target, question (identical to 55A)
// ──────────────────────────────────────────────

const SOURCE = "I want the business to grow, but I don't want to take on more risk.";
const CLARIFICATION_TARGET = "whether avoiding additional risk is a preference/trade-off or a hard constraint";
const CLARIFICATION_QUESTION = "Do you view avoiding additional risk as a hard constraint, or as a preference or trade-off?";

// ──────────────────────────────────────────────
// Three fixed cases — independent meaning-only and resolution calls
// ──────────────────────────────────────────────

const CASES = [
  {
    id: "Case 1 — Weak Priority",
    userAnswer: "Risk matters more to me.",
    humanReference: {
      answerMeaning: "risk has greater relative importance to the user",
      doesNotEstablish: ["that risk avoidance is a hard constraint", "that it is not a hard constraint"],
      targetResolved: false,
    },
  },
  {
    id: "Case 2 — Conditional Trade-Off",
    userAnswer: "I'd normally avoid more risk, but for the right opportunity I might accept some.",
    humanReference: {
      answerMeaning: "risk avoidance is normally preferred; additional risk may be accepted conditionally for the right opportunity",
      targetResolved: true,
    },
  },
  {
    id: "Case 3 — Non-Answer",
    userAnswer: "I'm not really sure.",
    humanReference: {
      answerMeaning: "the user remains uncertain",
      targetResolved: false,
    },
  },
];

// ──────────────────────────────────────────────
// Semantic evaluation helpers (manual review primary)
// ──────────────────────────────────────────────

function classifyMeaning(caseNum, meaningText) {
  const m = meaningText.toLowerCase().trim();

  // Case 1: weak priority — should NOT strengthen beyond relative importance
  if (caseNum === 1) {
    if (/hard.*constraint|not.*constraint|no.*need|don't.*need|absolute/i.test(m)) {
      return { classification: "meaning_strengthened", reasoning: `Meaning strengthened beyond relative priority: "${meaningText}"` };
    }
    if (/risk.*more.*important|risk.*matters.*more|greater.*priority|higher.*importance|relativ.*import/i.test(m)) {
      return { classification: "meaning_preserved", reasoning: `Weak priority preserved as relative importance without deciding constraint status.` };
    }
    if (m.length < 5) {
      return { classification: "meaning_lost", reasoning: `Meaning too brief to evaluate, possibly lost: "${meaningText}"` };
    }
    if (/risk/i.test(m) && !/more|greater|higher|priority|rival.*importance|trade.?off/i.test(m)) {
      return { classification: "meaning_strengthened", reasoning: `Meaning may have strengthened beyond relative priority: "${meaningText}"` };
    }
    return { classification: "meaning_preserved", reasoning: `Weak priority preserved (semantic review needed): "${meaningText}"` };
  }

  // Case 2: conditional trade-off — should preserve both normal preference and conditional exception
  if (caseNum === 2) {
    const hasConditionality = /but|condition|migh|would.*normally|normally.*avoid|exception|when/i.test(m);
    const hasNormalPreference = /normal.*avoid|normally.*risk|generally.*risk|usually.*risk|prefer.*risk|tend.*risk/i.test(m);
    const hasFlattening = /simply.*preference|risk is just|always avoid|i would.*not.*accept/i.test(m) && !hasConditionality;

    if (hasFlattening) {
      return { classification: "meaning_strengthened", reasoning: `Meaning flattened conditionality into flat statement: "${meaningText}"` };
    }
    if (hasConditionality && hasNormalPreference) {
      return { classification: "meaning_preserved", reasoning: `Conditional qualification preserved with normal preference and exception.` };
    }
    if (!hasConditionality && hasNormalPreference) {
      return { classification: "meaning_lost", reasoning: `Normal preference captured but conditionality lost: "${meaningText}"` };
    }
    return { classification: "meaning_preserved", reasoning: `Conditional meaning partially preserved (semantic review needed): "${meaningText}"` };
  }

  // Case 3: non-answer — should preserve simple uncertainty
  if (caseNum === 3) {
    if (/uncertain|not.*sure|don't.*know|no.*position|haven't.*decided|unsure/i.test(m)) {
      return { classification: "meaning_preserved", reasoning: `Uncertainty preserved from non-answer.` };
    }
    if (/risk|constraint|preference|avoid|should|must/i.test(m) && !/uncertain|unsure|not.*sure|don't.*know/i.test(m)) {
      return { classification: "meaning_strengthened", reasoning: `Meaning invented a position where user expressed uncertainty: "${meaningText}"` };
    }
    return { classification: "meaning_preserved", reasoning: `Non-answer meaning preserved (semantic review needed): "${meaningText}"` };
  }

  return { classification: "meaning_lost", reasoning: "Unrecognized case number" };
}

function classifyResolution(caseNum, result) {
  const { resolvedMeaning, targetResolved, remainingUncertainty } = result;
  const m = (resolvedMeaning ?? "").toLowerCase().trim();

  if (caseNum === 1) {
    if (targetResolved === true) {
      return { classification: "resolution_overresolved", reasoning: `Over-resolved weak answer: targetResolved=true. Meaning: "${resolvedMeaning}"` };
    }
    if (/not.*hard.*constraint|r.*isn't.*constraint|definitely.*not.*constraint/i.test(m)) {
      return { classification: "resolution_meaning_loss", reasoning: `Meaning strengthened during resolution judgement: "${resolvedMeaning}"` };
    }
    if (remainingUncertainty === null || remainingUncertainty === undefined) {
      return { classification: "resolution_overresolved", reasoning: `targetResolved=false but no remaining uncertainty — uncertainty erased.` };
    }
    return { classification: "resolution_correct", reasoning: `Weak answer correctly remained unresolved with remaining uncertainty.` };
  }

  if (caseNum === 2) {
    if (targetResolved !== true) {
      return { classification: "resolution_underresolved", reasoning: `Should be resolved but targetResolved=false: "${resolvedMeaning}"` };
    }
    const hasConditionality = /but|condition|migh|would.*normally|for.*opportunity/i.test(m);
    if (hasConditionality) {
      return { classification: "resolution_correct", reasoning: `Correctly resolved with conditional qualification preserved.` };
    }
    return { classification: "resolution_meaning_loss", reasoning: `Resolution correct but conditionality flattened: "${resolvedMeaning}"` };
  }

  if (caseNum === 3) {
    if (targetResolved === true) {
      return { classification: "resolution_overresolved", reasoning: `Over-resolved non-answer: targetResolved=true.` };
    }
    if (/invented|asserted.*position|would.*avoid/i.test(m)) {
      return { classification: "resolution_meaning_loss", reasoning: `Model invented meaning from non-answer: "${resolvedMeaning}"` };
    }
    if (remainingUncertainty === null || remainingUncertainty === undefined) {
      return { classification: "resolution_correct", reasoning: `Correctly unresolved, no spurious uncertainty.` };
    }
    return { classification: "resolution_correct", reasoning: `Correctly unresolved with remaining uncertainty.` };
  }

  return { classification: "resolution_correct", reasoning: "Unrecognized case number" };
}

// ──────────────────────────────────────────────
// Test suite
// ──────────────────────────────────────────────

describe("Experiment 55B — Separate Answer Meaning from Resolution Judgement", () => {
  const results = [];
  const timings = [];

  for (let i = 0; i < CASES.length; i++) {
    const c = CASES[i];
    const caseNum = i + 1;

    it(`${c.id} — Mode A: Meaning Only`, async () => {
      const start = Date.now();
      const result = await callMeaningOnly(SOURCE, CLARIFICATION_QUESTION, c.userAnswer);
      const elapsed = Date.now() - start;
      timings.push({ caseId: `${c.id}-A`, ms: elapsed });

      const meaningText = result.answerMeaning ?? "";
      const ev = classifyMeaning(caseNum, meaningText);

      results.push({
        caseNumber: caseNum,
        case: c,
        mode: "A",
        modelResult: result,
        rawMeaning: meaningText,
        classification: ev.classification,
        reasoning: ev.reasoning,
        timingMs: elapsed,
      });

      expect(result.answerMeaning).toBeDefined();
      expect(typeof result.answerMeaning).toBe("string");
      expect(result.answerMeaning.trim().length).toBeGreaterThan(0);
    }, 120000);

    it(`${c.id} — Mode B: Resolution`, async () => {
      const start = Date.now();
      const result = await callClarificationAnswerResolution(SOURCE, CLARIFICATION_TARGET, CLARIFICATION_QUESTION, c.userAnswer);
      const elapsed = Date.now() - start;
      timings.push({ caseId: `${c.id}-B`, ms: elapsed });

      const ev = classifyResolution(caseNum, result);

      results.push({
        caseNumber: caseNum,
        case: c,
        mode: "B",
        modelResult: result,
        classification: ev.classification,
        reasoning: ev.reasoning,
        timingMs: elapsed,
      });

      expect(result.resolvedMeaning).toBeDefined();
      expect(typeof result.resolvedMeaning).toBe("string");
      expect(result.targetResolved).toBeDefined();
      expect(typeof result.targetResolved).toBe("boolean");
    }, 120000);
  }

  it("55B: aggregate results and analysis", () => {
    const meaningCounts = { meaning_preserved: 0, meaning_strengthened: 0, meaning_lost: 0 };
    const resolutionCounts = { resolution_correct: 0, resolution_overresolved: 0, resolution_underresolved: 0, resolution_meaning_loss: 0 };

    const modeA = results.filter((r) => r.mode === "A");
    const modeB = results.filter((r) => r.mode === "B");

    for (const r of modeA) {
      meaningCounts[r.classification]++;
    }
    for (const r of modeB) {
      resolutionCounts[r.classification]++;
    }

    const totalMs = timings.reduce((s, t) => s + t.ms, 0);
    const msArr = timings.map((t) => t.ms);

    console.log("\n========== Experiment 55B Results ==========");
    console.log(`\nSource: ${SOURCE}`);
    console.log(`Target: ${CLARIFICATION_TARGET}`);
    console.log(`Question: ${CLARIFICATION_QUESTION}`);

    for (const r of modeA) {
      console.log(`\n--- Case ${r.caseNumber} Mode A (Meaning Only) ---`);
      console.log("Answer:", `"${r.case.userAnswer}"`);
      console.log("answerMeaning:", r.rawMeaning);
      console.log("Classification:", r.classification);
      console.log("Reasoning:", r.reasoning);
    }

    for (const r of modeB) {
      console.log(`\n--- Case ${r.caseNumber} Mode B (Resolution) ---`);
      console.log("Answer:", `"${r.case.userAnswer}"`);
      console.log("resolvedMeaning:", r.modelResult.resolvedMeaning);
      console.log("targetResolved:", r.modelResult.targetResolved);
      console.log("remainingUncertainty:", r.modelResult.remainingUncertainty ?? "null");
      console.log("Classification:", r.classification);
      console.log("Reasoning:", r.reasoning);
    }

    console.log("\n--- Meaning Classification Counts (Mode A) ---");
    Object.entries(meaningCounts).forEach(([k, v]) => console.log(`${k}: ${v}`));

    console.log("\n--- Resolution Classification Counts (Mode B) ---");
    Object.entries(resolutionCounts).forEach(([k, v]) => console.log(`${k}: ${v}`));

    console.log("\n--- Timing ---");
    console.log("Total live calls:", timings.length);
    console.log("Total time:", totalMs + "ms");
    console.log("Average:", (totalMs / timings.length).toFixed(1) + "ms per call");
    console.log("Fastest:", Math.min(...msArr) + "ms");
    console.log("Slowest:", Math.max(...msArr) + "ms");

    // Cross-mode comparison
    const crossMode = [];
    for (let i = 0; i < 3; i++) {
      const a = modeA[i];
      const b = modeB[i];
      crossMode.push({
        caseNumber: i + 1,
        answer: CASES[i].userAnswer,
        meaningClassification: a.classification,
        resolutionClassification: b.classification,
        meaningText: a.rawMeaning,
        resolutionText: b.modelResult.resolvedMeaning,
      });
    }

    console.log("\n--- Cross-Mode Comparison ---");
    for (const c of crossMode) {
      const meaningOk = c.meaningClassification === "meaning_preserved";
      const resolutionOk = c.resolutionClassification === "resolution_correct" || c.resolutionClassification === "resolution_meaning_loss";
      console.log(`\nCase ${c.caseNumber}: "${c.answer}"`);
      console.log(`  Meaning: ${c.meaningClassification} — "${c.meaningText}"`);
      console.log(`  Resolution: ${c.resolutionClassification} — "${c.resolutionText}"`);

      if (meaningOk && c.resolutionClassification !== "resolution_correct") {
        console.log(`  => Meaning preserved in Mode A but resolution judgement introduced a problem.`);
      } else if (!meaningOk) {
        console.log(`  => Meaning was already distorted before the resolution judgement.`);
      }
    }

    const q1 = modeA[0].classification === "meaning_preserved";
    const q2 = modeB[0].classification !== "resolution_correct" && (modeB[0].classification === "resolution_overresolved" || modeB[0].classification === "resolution_meaning_loss");
    const q3 = (/(?:preserved.*conditionality|conditional.*qualification.*preserved)/.test(modeA[1].reasoning)) || modeA[1].classification === "meaning_preserved";
    const q4 = modeB[1].classification === "resolution_correct" || /flattened/i.test(modeB[1].reasoning);
    const q5 = modeA[2].classification === "meaning_preserved";
    const q6 = modeB[2].classification !== "resolution_overresolved";

    console.log("\n--- Required Questions ---");
    console.log("Q1 (Case 1 meaning preserved as relative priority?):", q1 ? "Yes" : "No — " + modeA[0].reasoning);
    console.log("Q2 (Case 1 resolution over-resolved again?):", q2 ? "Yes" : "No");
    console.log("Q3 (Case 2 meaning preserved conditional qualification?):", q3 ? "Yes" : "Need review — " + modeA[1].reasoning);
    console.log("Q4 (Case 2 resolution preserve or flatten conditionality?):", /preserve/i.test(modeB[1].reasoning) ? "Preserved" : /flatten/i.test(modeB[1].reasoning) ? "Flattened" : "Unclear — " + modeB[1].reasoning);
    console.log("Q5 (Case 3 meaning preserved uncertainty?):", q5 ? "Yes" : "No");
    console.log("Q6 (Case 3 resolution correctly unresolved?):", q6 ? "Yes" : "No");

    const anyMeaningDistortedBeforeResolution = crossMode.some(
      (c) => c.meaningClassification !== "meaning_preserved"
    );
    const meaningPreservedButResolutionProblems = crossMode.some(
      (c) => c.meaningClassification === "meaning_preserved" && c.resolutionClassification !== "resolution_correct"
    );

    console.log("\n--- Key Findings ---");
    console.log("Meaning already distorted before resolution:", anyMeaningDistortedBeforeResolution);
    console.log("Meaning preserved in Mode A but resolution introduced problem:", meaningPreservedButResolutionProblems);

    results.summary = {
      meaningCounts,
      resolutionCounts,
      totalTimeMs: totalMs,
      totalCalls: timings.length,
      averageMs: (totalMs / timings.length).toFixed(1),
      fastestMs: Math.min(...msArr),
      slowestMs: Math.max(...msArr),
      crossMode,
      anyMeaningDistortedBeforeResolution,
      meaningPreservedButResolutionProblems,
    };

    console.log("\n========== End of Experiment 55B ==========\n");
  });
});