confidence-engine/tests/reconstruction/semantic-preserved-meaning-resolution.test.js

import { describe, it, expect } from "vitest";
import { config } from "dotenv";
import path from "path";
import { fileURLToPath } from "url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
config({ path: path.resolve(__dirname, "../../.env.local") });

const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL;
const OLLAMA_MODEL = process.env.OLLAMA_MODEL;

if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) {
  throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local");
}

// ──────────────────────────────────────────────
// Stage 1 — Preserve Answer Meaning (exact Mode A from 55B)
// ──────────────────────────────────────────────

async function callPreserveMeaning(source, clarificationQuestion, userAnswer) {
  const instruction = `State only what the user's answer establishes in relation to the clarification question. Preserve uncertainty, conditionality, and qualification exactly as supplied. Do not decide whether the clarification target is resolved. Do not infer what the user did not say. Do not recommend action or generate another question.

Return valid JSON only in this shape:
{
  "answerMeaning": "short statement"
}`;

  const messages = [
    { role: "system", content: instruction.trim() },
    {
      role: "user",
      content: `Source: ${JSON.stringify(source)}

Clarification question: ${clarificationQuestion}

User's answer: ${userAnswer}`,
    },
  ];

  const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model: OLLAMA_MODEL,
      messages,
      format: "json",
      stream: false,
    }),
  });

  if (!res.ok) {
    throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
  }

  const data = await res.json();
  const rawContent = data.message?.content ?? "";
  const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");

  return JSON.parse(cleaned.trim());
}

// ──────────────────────────────────────────────
// Stage 2 — Resolve From Preserved Meaning (no raw answer)
// ──────────────────────────────────────────────

async function callResolveFromPreservedMeaning(source, clarificationTarget, preservedAnswerMeaning) {
  const instruction = `Decide whether the supplied preserved answer meaning settles the clarification target. Treat the preserved meaning as the full extent of what has been established — do not strengthen, simplify, or reinterpret it. Mark targetResolved true only when that preserved meaning settles the target. If any part of the target remains unresolved, preserve that uncertainty. Keep qualifications and conditions intact.

Return valid JSON only in this shape:
{
  "resolvedMeaning": "short statement",
  "targetResolved": true,
  "remainingUncertainty": null
}`;

  const messages = [
    { role: "system", content: instruction.trim() },
    {
      role: "user",
      content: `Source: ${JSON.stringify(source)}

Clarification target: ${clarificationTarget}

Preserved answer meaning: "${preservedAnswerMeaning}"`,
    },
  ];

  const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model: OLLAMA_MODEL,
      messages,
      format: "json",
      stream: false,
    }),
  });

  if (!res.ok) {
    throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
  }

  const data = await res.json();
  const rawContent = data.message?.content ?? "";
  const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");

  return JSON.parse(cleaned.trim());
}

// ──────────────────────────────────────────────
// Fixed source / target / question (identical to 55A/55B)
// ──────────────────────────────────────────────

const SOURCE = "I want the business to grow, but I don't want to take on more risk.";
const CLARIFICATION_TARGET = "whether avoiding additional risk is a preference/trade-off or a hard constraint";
const CLARIFICATION_QUESTION = "Do you view avoiding additional risk as a hard constraint, or as a preference or trade-off?";

// ──────────────────────────────────────────────
// Three cases
// ──────────────────────────────────────────────

const CASES = [
  {
    id: "Case 1 — Weak Priority",
    userAnswer: "Risk matters more to me.",
    humanReference: {
      stage1Meaning: "risk has greater relative importance",
      stage2TargetResolved: false,
      note: "relative importance does not establish whether risk avoidance is a hard constraint or flexible preference/trade-off",
    },
  },
  {
    id: "Case 2 — Conditional Trade-Off",
    userAnswer: "I'd normally avoid more risk, but for the right opportunity I might accept some.",
    humanReference: {
      stage1Meaning: "normally prefers avoiding additional risk; may accept additional risk conditionally for the right opportunity",
      stage2TargetResolved: true,
      note: "establishes that avoiding all additional risk is not an absolute hard constraint; resolved meaning must retain conditional qualification",
    },
  },
  {
    id: "Case 3 — Non-Answer",
    userAnswer: "I'm not really sure.",
    humanReference: {
      stage1Meaning: "the user remains uncertain",
      stage2TargetResolved: false,
      note: "preference-versus-hard-constraint distinction still unresolved",
    },
  },
];

// ──────────────────────────────────────────────
// Semantic evaluation helpers (manual review primary)
// ──────────────────────────────────────────────

function classifyMeaning(caseNum, meaningText) {
  const m = meaningText.toLowerCase().trim();

  if (caseNum === 1) {
    if (/hard.*constraint|not.*constraint|no.*need|don't.*need|absolute/i.test(m)) {
      return { classification: "meaning_strengthened", reasoning: `Meaning strengthened beyond relative priority: "${meaningText}"` };
    }
    if (/risk.*more.*important|risk.*matters.*more|greater.*priority|higher.*importance|relativ.*import/i.test(m)) {
      return { classification: "meaning_preserved", reasoning: `Weak priority preserved as relative importance without deciding constraint status.` };
    }
    if (m.length < 5) {
      return { classification: "meaning_lost", reasoning: `Meaning too brief to evaluate, possibly lost: "${meaningText}"` };
    }
    if (/risk/i.test(m) && !/more|greater|higher|priority|rival.*importance|trade.?off/i.test(m)) {
      return { classification: "meaning_strengthened", reasoning: `Meaning may have strengthened beyond relative priority: "${meaningText}"` };
    }
    return { classification: "meaning_preserved", reasoning: `Weak priority preserved (semantic review needed): "${meaningText}"` };
  }

  if (caseNum === 2) {
    const hasConditionality = /but|condition|migh|would.*normally|normally.*avoid|exception|when/i.test(m);
    const hasNormalPreference = /normal.*avoid|normally.*risk|generally.*risk|usually.*risk|prefer.*risk|tend.*risk/i.test(m);
    const hasFlattening = /simply.*preference|risk is just|always avoid|i would.*not.*accept/i.test(m) && !hasConditionality;

    if (hasFlattening) {
      return { classification: "meaning_strengthened", reasoning: `Meaning flattened conditionality into flat statement: "${meaningText}"` };
    }
    if (hasConditionality && hasNormalPreference) {
      return { classification: "meaning_preserved", reasoning: `Conditional qualification preserved with normal preference and exception.` };
    }
    if (!hasConditionality && hasNormalPreference) {
      return { classification: "meaning_lost", reasoning: `Normal preference captured but conditionality lost: "${meaningText}"` };
    }
    return { classification: "meaning_preserved", reasoning: `Conditional meaning partially preserved (semantic review needed): "${meaningText}"` };
  }

  if (caseNum === 3) {
    if (/uncertain|not.*sure|don't.*know|no.*position|haven't.*decided|unsure/i.test(m)) {
      return { classification: "meaning_preserved", reasoning: `Uncertainty preserved from non-answer.` };
    }
    if (/risk|constraint|preference|avoid|should|must/i.test(m) && !/uncertain|unsure|not.*sure|don't.*know/i.test(m)) {
      return { classification: "meaning_strengthened", reasoning: `Meaning invented a position where user expressed uncertainty: "${meaningText}"` };
    }
    return { classification: "meaning_preserved", reasoning: `Non-answer meaning preserved (semantic review needed): "${meaningText}"` };
  }

  return { classification: "meaning_lost", reasoning: "Unrecognized case number" };
}

function classifyResolution(caseNum, result) {
  const { resolvedMeaning, targetResolved, remainingUncertainty } = result;
  const m = (resolvedMeaning ?? "").toLowerCase().trim();

  if (caseNum === 1) {
    if (targetResolved === true) {
      return { classification: "resolution_overresolved", reasoning: `Over-resolved weak answer: targetResolved=true. Meaning: "${resolvedMeaning}"` };
    }
    if (/not.*hard.*constraint|r.*isn't.*constraint|definitely.*not.*constraint/i.test(m)) {
      return { classification: "resolution_meaning_loss", reasoning: `Meaning strengthened during resolution judgement: "${resolvedMeaning}"` };
    }
    if (remainingUncertainty === null || remainingUncertainty === undefined) {
      return { classification: "resolution_overresolved", reasoning: `targetResolved=false but no remaining uncertainty — uncertainty erased.` };
    }
    return { classification: "resolution_correct", reasoning: `Weak answer correctly remained unresolved with remaining uncertainty.` };
  }

  if (caseNum === 2) {
    if (targetResolved !== true) {
      return { classification: "resolution_underresolved", reasoning: `Should be resolved but targetResolved=false: "${resolvedMeaning}"` };
    }
    const hasConditionality = /but|condition|migh|would.*normally|for.*opportunity/i.test(m);
    if (hasConditionality) {
      return { classification: "resolution_correct", reasoning: `Correctly resolved with conditional qualification preserved.` };
    }
    return { classification: "resolution_meaning_loss", reasoning: `Resolution correct but conditionality flattened: "${resolvedMeaning}"` };
  }

  if (caseNum === 3) {
    if (targetResolved === true) {
      return { classification: "resolution_overresolved", reasoning: `Over-resolved non-answer: targetResolved=true.` };
    }
    if (/invented|asserted.*position|would.*avoid/i.test(m)) {
      return { classification: "resolution_meaning_loss", reasoning: `Model invented meaning from non-answer: "${resolvedMeaning}"` };
    }
    if (remainingUncertainty === null || remainingUncertainty === undefined) {
      return { classification: "resolution_correct", reasoning: `Correctly unresolved, no spurious uncertainty.` };
    }
    return { classification: "resolution_correct", reasoning: `Correctly unresolved with remaining uncertainty.` };
  }

  return { classification: "resolution_correct", reasoning: "Unrecognized case number" };
}

// ──────────────────────────────────────────────
// Test suite — the core experiment
// ──────────────────────────────────────────────

describe("Experiment 55C — Resolution From Preserved Answer Meaning", () => {
  const results = [];
  const timings = [];

  for (let i = 0; i < CASES.length; i++) {
    const c = CASES[i];
    const caseNum = i + 1;

    // Stage 1: preserve meaning from raw answer
    it(`${c.id} — Stage 1: Preserve Meaning`, async () => {
      const start = Date.now();
      const stage1Result = await callPreserveMeaning(SOURCE, CLARIFICATION_QUESTION, c.userAnswer);
      const elapsed = Date.now() - start;
      timings.push({ caseId: `${c.id}-S1`, ms: elapsed });

      const meaningText = stage1Result.answerMeaning ?? "";
      const ev = classifyMeaning(caseNum, meaningText);

      results.push({
        caseNumber: caseNum,
        case: c,
        stage: 1,
        rawAnswer: c.userAnswer,
        stage1Meaning: meaningText,
        stage1Result: stage1Result,
        classification: ev.classification,
        reasoning: ev.reasoning,
        timingMs: elapsed,
      });

      expect(stage1Result.answerMeaning).toBeDefined();
      expect(typeof stage1Result.answerMeaning).toBe("string");
      expect(stage1Result.answerMeaning.trim().length).toBeGreaterThan(0);
    }, 120000);

    // Stage 2: resolve using ACTUAL Stage 1 output (not human reference)
    it(`${c.id} — Stage 2: Resolve From Preserved Meaning`, async () => {
      const start = Date.now();
      const stage1Meaning = results.find((r) => r.caseNumber === caseNum && r.stage === 1).stage1Meaning;

      const stage2Result = await callResolveFromPreservedMeaning(
        SOURCE,
        CLARIFICATION_TARGET,
        stage1Meaning
      );
      const elapsed = Date.now() - start;
      timings.push({ caseId: `${c.id}-S2`, ms: elapsed });

      const ev = classifyResolution(caseNum, stage2Result);

      results.push({
        caseNumber: caseNum,
        case: c,
        stage: 2,
        rawAnswer: c.userAnswer,
        stage1Meaning: stage1Meaning,
        stage2Input: stage1Meaning,
        stage2Result: stage2Result,
        resolutionCorrect: ev.classification === "resolution_correct",
        classification: ev.classification,
        reasoning: ev.reasoning,
        timingMs: elapsed,
      });

      expect(stage2Result.resolvedMeaning).toBeDefined();
      expect(typeof stage2Result.resolvedMeaning).toBe("string");
      expect(stage2Result.targetResolved).toBeDefined();
      expect(typeof stage2Result.targetResolved).toBe("boolean");
    }, 120000);
  }

  // ────────────────────────────────────────────
  // Aggregate analysis
  // ────────────────────────────────────────────

  it("55C: aggregate results and analysis", () => {
    const meaningCounts = { meaning_preserved: 0, meaning_strengthened: 0, meaning_lost: 0 };
    const resolutionCounts = {
      resolution_correct: 0,
      resolution_overresolved: 0,
      resolution_underresolved: 0,
      resolution_meaning_loss: 0,
    };

    const stage1Results = results.filter((r) => r.stage === 1);
    const stage2Results = results.filter((r) => r.stage === 2);

    for (const r of stage1Results) {
      meaningCounts[r.classification]++;
    }
    for (const r of stage2Results) {
      resolutionCounts[r.classification]++;
    }

    const totalMs = timings.reduce((s, t) => s + t.ms, 0);
    const msArr = timings.map((t) => t.ms);

    console.log("\n========== Experiment 55C Results ==========");
    console.log(`\nSource: ${SOURCE}`);
    console.log(`Target: ${CLARIFICATION_TARGET}`);
    console.log(`Question: ${CLARIFICATION_QUESTION}`);

    // Stage 1 results
    for (const r of stage1Results) {
      console.log(`\n--- Case ${r.caseNumber} Stage 1 (Preserve Meaning) ---`);
      console.log("Raw answer:", `"${r.rawAnswer}"`);
      console.log("answerMeaning:", r.stage1Meaning);
      console.log("Classification:", r.classification);
      console.log("Reasoning:", r.reasoning);
    }

    // Stage 2 results
    for (const r of stage2Results) {
      console.log(`\n--- Case ${r.caseNumber} Stage 2 (Resolve From Preserved Meaning) ---`);
      console.log("Raw answer:", `"${r.rawAnswer}"`);
      console.log("Stage 1 input (actual preserved meaning):", `"${r.stage2Input}"`);
      console.log("resolvedMeaning:", r.stage2Result.resolvedMeaning);
      console.log("targetResolved:", r.stage2Result.targetResolved);
      console.log("remainingUncertainty:", r.stage2Result.remainingUncertainty ?? "null");
      console.log("Classification:", r.classification);
      console.log("Reasoning:", r.reasoning);
    }

    // Chaining analysis — does Stage 2 strengthen beyond actual Stage 1?
    for (let i = 0; i < stage1Results.length; i++) {
      const s1 = stage1Results[i];
      const s2 = stage2Results[i];
      console.log(`\n--- Chaining: Case ${i + 1} ---`);
      console.log("Stage 1 meaning:", `"${s1.stage1Meaning}"`);
      console.log("Stage 2 resolvedMeaning:", `"${s2.stage2Result.resolvedMeaning}"`);

      // Check if Stage 2 strengthened beyond Stage 1
      const s1Lower = s1.stage1Meaning.toLowerCase();
      const s2Lower = s2.stage2Result.resolvedMeaning.toLowerCase();

      // Stronger language indicators in Stage 2 not present in Stage 1
      const strongInS2 = /definitely|clearly|established as|proves|confirms|shows.*not/i.test(s2Lower);
      const absentInS1 = !/definitely|clearly|established as|proves|confirms|shows.*not/i.test(s1Lower);
      if (strongInS2 && absentInS1) {
        console.log("  WARNING: Stage 2 may have strengthened beyond Stage 1 meaning.");
      } else {
        console.log("  Stage 2 did not strengthen beyond Stage 1 (semantic check).");
      }

      // Check if Stage 2 flattened a condition present in Stage 1
      const s1HasConditionality = /but|condition|migh|would.*normally|normally.*avoid|exception|when/
        .test(s1Lower);
      const s2HasConditionality = /but|condition|migh|would.*normally|for.*opportunity/
        .test(s2Lower);
      if (s1HasConditionality && !s2HasConditionality) {
        console.log("  WARNING: Stage 2 flattened a condition present in Stage 1.");
      } else if (s1HasConditionality && s2HasConditionality) {
        console.log("  Stage 2 retained the condition from Stage 1.");
      }
    }

    console.log("\n--- Meaning Classification Counts (Stage 1) ---");
    Object.entries(meaningCounts).forEach(([k, v]) => console.log(`${k}: ${v}`));

    console.log("\n--- Resolution Classification Counts (Stage 2) ---");
    Object.entries(resolutionCounts).forEach(([k, v]) => console.log(`${k}: ${v}`));

    console.log("\n--- Timing ---");
    console.log("Total live calls:", timings.length);
    console.log("Total time:", totalMs + "ms");
    console.log("Average:", (totalMs / timings.length).toFixed(1) + "ms per call");
    console.log("Fastest:", Math.min(...msArr) + "ms");
    console.log("Slowest:", Math.max(...msArr) + "ms");

    // Required questions
    const q1 = stage1Results[0].classification === "meaning_preserved";
    const q2 = stage2Results[0].classification === "resolution_correct";
    const q3 = stage1Results[1].classification === "meaning_preserved";
    const q4 = stage2Results[1].classification === "resolution_correct" ||
      /flattened/i.test(stage2Results[1].reasoning);
    const q5 = stage1Results[2].classification === "meaning_preserved";
    const q6 = stage2Results[0].classification !== "resolution_overresolved" &&
      stage2Results[0].classification !== "resolution_meaning_loss";

    console.log("\n--- Required Questions ---");
    console.log("Q1 (Case 1 Stage 1 preserved only relative priority?):", q1 ? "Yes" : "No — " + stage1Results[0].reasoning);
    console.log("Q2 (Case 1 Stage 2 remained unresolved?):", q2 ? "Yes" : "No — " + stage2Results[0].reasoning);
    console.log("Q3 (Case 2 Stage 1 preserved conditional qualification?):", q3 ? "Yes" : "No — " + stage1Results[1].reasoning);
    console.log("Q4 (Case 2 Stage 2 retained qualification while resolving?):",
      /preserve/i.test(stage2Results[1].reasoning) ? "Preserved" :
        /flatten/i.test(stage2Results[1].reasoning) ? "Flattened" :
          "Unclear — " + stage2Results[1].reasoning);
    console.log("Q5 (Case 3 preserved uncertainty through both stages?):",
      q5 && stage2Results[2].classification === "resolution_correct" ? "Yes" : "No");
    console.log("Q6 (Did any Stage 2 strengthen actual Stage 1?):", q6 ? "No observed strengthening" : "Possible — check chaining analysis above");

    // Questions 10-12: always No
    console.log("\n--- Required Answers to Critical Questions ---");
    console.log("Q10 (proves two-stage production design required?): No");
    console.log("Q11 (establishes graph representation?): No");
    console.log("Q12 (establishes Behaviour Selection changes?): No");

    // Evidence summary
    const meaningPreservedCount = meaningCounts.meaning_preserved;
    const resolutionCorrectCount = resolutionCounts.resolution_correct;
    const resolutionMeaningLossCount = resolutionCounts.resolution_meaning_loss;
    const anyStrengthened = stage2Results.some((s2, idx) => {
      const s1 = stage1Results[idx];
      const s1Lower = s1.stage1Meaning.toLowerCase();
      const s2Lower = s2.stage2Result.resolvedMeaning.toLowerCase();
      return /definitely|clearly|established as|proves|confirms/i.test(s2Lower) &&
        !/definitely|clearly|established as|proves|confirms/i.test(s1Lower);
    });
    const anyFlattened = stage2Results.some((s2, idx) => {
      const s1 = stage1Results[idx];
      const s1Lower = s1.stage1Meaning.toLowerCase();
      const s2Lower = s2.stage2Result.resolvedMeaning.toLowerCase();
      const s1HasCond = /but|condition|migh|would.*normally|normally.*avoid/i.test(s1Lower);
      const s2HasCond = /but|condition|migh|would.*normally|for.*opportunity/i.test(s2Lower);
      return s1HasCond && !s2HasCond;
    });

    console.log("\n--- Evidence Summary ---");
    console.log("Meaning preserved (Stage 1):", meaningPreservedCount, "/ 3");
    console.log("Resolution correct:", resolutionCorrectCount, "/ 3");
    console.log("Any Stage 2 strengthened beyond actual Stage 1:", anyStrengthened);
    console.log("Any Stage 2 flattened a condition from Stage 1:", anyFlattened);

    results.summary = {
      meaningCounts,
      resolutionCounts,
      totalTimeMs: totalMs,
      totalCalls: timings.length,
      averageMs: (totalMs / timings.length).toFixed(1),
      fastestMs: Math.min(...msArr),
      slowestMs: Math.max(...msArr),
      anyStrengthenedBeyondStage1: anyStrengthened,
      anyFlattenedConditionality: anyFlattened,
      stage1Results,
      stage2Results,
    };

    console.log("\n========== End of Experiment 55C ==========\n");
  });
});