confidence-engine/tests/reconstruction/semantic-clarification-stated-vs-inferred.test.js

import { describe, it, expect } from "vitest";
import { config } from "dotenv";
import path from "path";
import { fileURLToPath } from "url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
config({ path: path.resolve(__dirname, "../../.env.local") });

const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL;
const OLLAMA_MODEL = process.env.OLLAMA_MODEL;

if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) {
  throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local");
}

// Semantic instruction: stated vs inferred separation
const SEMANTIC_INSTRUCTION = `State only what the user's answer directly establishes in statedMeaning. Preserve uncertainty, qualification, and conditionality. Do not turn relative importance into a hard boundary or the absence of one unless the user actually says so. If there is a plausible implication that goes beyond what the answer directly establishes, place it only in possibleInference. Do not decide whether the clarification target is resolved. Do not recommend action or generate another question.`;

const CLARIFICATION_QUESTION = "Do you view avoiding additional risk as a hard constraint, or as a preference or trade-off?";
const CLARIFICATION_TARGET = "whether avoiding additional risk is a preference/trade-off or a hard constraint";

// Four fixed answers — one live call each
const CASES = [
  {
    id: "Case 1 - Weak Priority",
    userAnswer: "Risk matters more to me.",
    humanReference: {
      statedMeaning: "risk has greater relative importance to the user",
      possibleInferenceAcceptable: "this may indicate a strong preference toward avoiding risk",
      notes: "must NOT establish hard constraint or not-a-hard-constraint in statedMeaning",
    },
  },
  {
    id: "Case 2 - Conditional Trade-Off",
    userAnswer: "I'd normally avoid more risk, but for the right opportunity I might accept some.",
    humanReference: {
      statedMeaning: "normally prefers avoiding additional risk; may accept some conditionally for the right opportunity",
      possibleInferenceAcceptable: null,
      notes: "must preserve both normal preference AND conditional exception; must directly establish that avoiding all additional risk is not absolute",
    },
  },
  {
    id: "Case 3 - Explicit Hard Constraint",
    userAnswer: "It's a hard constraint. I don't want any increase in risk.",
    humanReference: {
      statedMeaning: "avoiding additional risk is a hard constraint / no increase in risk is acceptable",
      possibleInferenceAcceptable: null,
      notes: "positive control for explicit meaning; possibleInference should normally be null",
    },
  },
  {
    id: "Case 4 - Non-Answer",
    userAnswer: "I'm not really sure.",
    humanReference: {
      statedMeaning: "the user remains uncertain",
      possibleInferenceAcceptable: null,
      notes: "must NOT invent a preference, constraint, or likely leaning; possibleInference should normally be null",
    },
  },
];

// One live call per case - stated vs inferred separation
async function callStatedVsInferred(clarificationTarget, clarificationQuestion, userAnswer) {
  const instruction = `State only what the user's answer directly establishes in statedMeaning. Preserve uncertainty, qualification, and conditionality. Do not turn relative importance into a hard boundary or the absence of one unless the user actually says so. If there is a plausible implication that goes beyond what the answer directly establishes, place it only in possibleInference. Do not decide whether the clarification target is resolved. Do not recommend action or generate another question.

Return valid JSON only in this shape:
{
  "statedMeaning": "short statement",
  "possibleInference": "short statement or null"
}`;

  const messages = [
    { role: "system", content: instruction.trim() },
    {
      role: "user",
      content: `Clarification target context: ${clarificationTarget}

Clarification question: ${clarificationQuestion}

User's answer: ${userAnswer}`,
    },
  ];

  const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({
      model: OLLAMA_MODEL,
      messages,
      format: "json",
      stream: false,
    }),
  });

  if (!res.ok) {
    throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
  }

  const data = await res.json();
  const rawContent = data.message?.content ?? "";
  const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");

  return JSON.parse(cleaned.trim());
}

// Semantic evaluation helpers (manual review primary)
function classifyStatedMeaning(caseNum, statedMeaningText) {
  const m = (statedMeaningText ?? "").toLowerCase().trim();

  if (caseNum === 1) {
    // Weak priority: must NOT establish hard constraint or not-a-hard-constraint
    if (/hard.*constraint|not.*a.*hard.*constraint|non.?negotiable|definitively.*not|no.*need.*to.*worry|absolute.*bound/i.test(m)) {
      return { classification: "stated_meaning_strengthened", reasoning: `Weak priority strengthened into constraint language in statedMeaning: "${statedMeaningText}"` };
    }
    if (/greater.*importance|more.*important|matters.*more|risk.*has.*weight|relativ.*priority|higher.*concern/i.test(m)) {
      return { classification: "stated_meaning_preserved", reasoning: `Weak priority preserved as relative importance only in statedMeaning.` };
    }
    if (m.length < 5) {
      return { classification: "stated_meaning_lost", reasoning: `statedMeaning too brief to evaluate: "${statedMeaningText}"` };
    }
    // Partial match - needs semantic review
    return { classification: "stated_meaning_preserved", reasoning: `Weak priority may be preserved (semantic review needed): "${statedMeaningText}"` };
  }

  if (caseNum === 2) {
    const hasConditionality = /but|condition|migh|would.*normally|normally.*avoid|tend.*to.*avoid|generally.*prefer/i.test(m);
    if (!hasConditionality) {
      return { classification: "stated_meaning_lost", reasoning: `Conditional qualification lost in statedMeaning - flattened to flat preference: "${statedMeaningText}"` };
    }
    const hasNormalPreference = /normal|generally|usually|tend|prefer.*avoid/i.test(m);
    const hasConditionalAcceptance = /accept.*some|might.*accept|conditional.*accept|opportunity.*might|when.*right.*opportun/i.test(m);
    if (hasNormalPreference && hasConditionalAcceptance) {
      return { classification: "stated_meaning_preserved", reasoning: `Both normal preference and conditional exception preserved in statedMeaning.` };
    }
    if (!hasNormalPreference || !hasConditionalAcceptance) {
      return { classification: "stated_meaning_lost", reasoning: `One side of conditionality missing from statedMeaning: "${statedMeaningText}"` };
    }
  }

  if (caseNum === 3) {
    // Explicit hard constraint: must establish it directly
    if (/hard.*constraint|definitive.*no.*increase|no.*acceptable|absolute.*boundary|won't.*accept.*risk|must.*avoid.*any/i.test(m)) {
      return { classification: "stated_meaning_preserved", reasoning: `Explicit hard constraint preserved in statedMeaning.` };
    }
    if (/uncertain|don't.*know|not.*sure/i.test(m)) {
      return { classification: "stated_meaning_lost", reasoning: `Explicit meaning lost - replaced with uncertainty: "${statedMeaningText}"` };
    }
    return { classification: "stated_meaning_preserved", reasoning: `Hard constraint may be preserved (semantic review needed): "${statedMeaningText}"` };
  }

  if (caseNum === 4) {
    // Non-answer: must establish only uncertainty, no invented preference
    if (/uncertain|not.*sure|don't.*know|no.*position|haven't.*decided|unsure/i.test(m)) {
      return { classification: "stated_meaning_preserved", reasoning: `Uncertainty preserved in statedMeaning.` };
    }
    if (/risk.*avoid|preference.*for|risk.*matters|should.*avoid|would.*prefer/i.test(m) && !/uncertain|unsure|don't.*know/i.test(m)) {
      return { classification: "stated_meaning_strengthened", reasoning: `Invented a preference/constraint from non-answer in statedMeaning: "${statedMeaningText}"` };
    }
    if (m.length < 5) {
      return { classification: "stated_meaning_lost", reasoning: `statedMeaning too brief to evaluate: "${statedMeaningText}"` };
    }
    return { classification: "stated_meaning_preserved", reasoning: `Uncertainty may be preserved (semantic review needed): "${statedMeaningText}"` };
  }

  return { classification: "stated_meaning_lost", reasoning: "Unrecognized case number" };
}

function classifyInferenceSeparation(caseNum, statedMeaningText, possibleInferenceText) {
  const sm = (statedMeaningText ?? "").toLowerCase().trim();
  const pi = (possibleInferenceText ?? null);
  const piStr = typeof pi === "string" ? pi.toLowerCase().trim() : null;

  if (caseNum === 1) {
    // Weak priority: statedMeaning should only have relative importance
    const hasRelativeImportanceInStated = /greater.*importance|more.*important|matters.*more|risk.*has.*weight|relativ/i.test(sm);
    const hasConstraintLanguageInStated = /hard.*constraint|not.*a.*hard.*constraint|non.?negotiable|absolute.*bound/i.test(sm);

    if (!hasRelativeImportanceInStated && !hasConstraintLanguageInStated) {
      return { classification: "unnecessary_inference", reasoning: `statedMeaning lacks relative importance and possibleInference may add unnecessary implication: stated="${sm}" inferred="${piStr}"` };
    }

    if (hasConstraintLanguageInStated) {
      return { classification: "inference_leaked_into_stated", reasoning: `Stronger constraint language leaked into statedMeaning where only relative importance should appear: "${statedMeaningText}"` };
    }

    if (piStr !== null && piStr.length > 0) {
      const isReasonableImplication = /may.*indicate|could.*suggest|might.*lean|potentially.*stronger|i.*leans.*toward/i.test(piStr);
      if (isReasonableImplication) {
        return { classification: "inference_cleanly_separated", reasoning: `possibleInference contains reasonable implication (may indicate/could suggest) separate from statedMeaning's relative importance.` };
      }
      return { classification: "inference_cleanly_separated", reasoning: `possibleInference contains an implication beyond statedMeaning. Review whether it crosses into unnecessary inference: "${piStr}"` };
    }

    if (pi === null) {
      return { classification: "no_inference_needed", reasoning: `statedMeaning preserved relative importance only; no additional inference needed.` };
    }

    return { classification: "unnecessary_inference", reasoning: `possibleInference may be unnecessary where statedMeaning is clean: "${piStr}"` };
  }

  if (caseNum === 2) {
    if (pi === null || piStr.length === 0) {
      return { classification: "no_inference_needed", reasoning: `No inference needed - statedMeaning captures the full answer.` };
    }
    const isReasonable = /may.*indicate|could.*suggest|might.*imply|potentially/i.test(piStr);
    if (isReasonable) {
      return { classification: "inference_cleanly_separated", reasoning: `possibleInference contains a reasonable implication, cleanly separated from stated meaning.` };
    }
    const overlaps = piStr.split(/\s+/).some(w => w.length > 4 && sm.includes(w));
    if (overlaps && !isReasonable) {
      return { classification: "unnecessary_inference", reasoning: `possibleInference adds little beyond statedMeaning: "${piStr}"` };
    }
    return { classification: "inference_cleanly_separated", reasoning: `possibleInference present and separated. Semantic review recommended: "${piStr}"` };
  }

  if (caseNum === 3) {
    // Explicit hard constraint: possibleInference should normally be null
    if (pi === null || piStr.length === 0) {
      return { classification: "no_inference_needed", reasoning: `No inference needed - statedMeaning is explicit.` };
    }
    return { classification: "unnecessary_inference", reasoning: `possibleInference should normally be null for explicit hard constraint. It adds: "${piStr}"` };
  }

  if (caseNum === 4) {
    // Non-answer: possibleInference should normally be null
    if (pi === null || piStr.length === 0) {
      return { classification: "no_inference_needed", reasoning: `No inference needed for non-answer.` };
    }
    const inventsLeaning = /would.*prefer|likely.*to.*avoid|probably.*want|tends.*toward|most.*people.*would/i.test(piStr);
    if (inventsLeaning) {
      return { classification: "unnecessary_inference", reasoning: `possibleInference invents a leaning from non-answer: "${piStr}"` };
    }
    return { classification: "unnecessary_inference", reasoning: `possibleInference present for non-answer where none is warranted: "${piStr}"` };
  }

  return { classification: "no_inference_needed", reasoning: "Unrecognized case number" };
}

// Test suite - Experiment 55D
describe("Experiment 55D - Separate Stated Clarification Meaning from Inference", () => {
  const results = [];
  const timings = [];

  for (let i = 0; i < CASES.length; i++) {
    const c = CASES[i];
    const caseNum = i + 1;

    it(`${c.id} - One call: stated vs inferred separation`, async () => {
      const start = Date.now();
      const result = await callStatedVsInferred(
        CLARIFICATION_TARGET,
        CLARIFICATION_QUESTION,
        c.userAnswer
      );
      const elapsed = Date.now() - start;
      timings.push({ caseId: c.id, ms: elapsed });

      const statedMeaning = result.statedMeaning ?? "";
      const possibleInference = result.possibleInference ?? null;

      const smClass = classifyStatedMeaning(caseNum, statedMeaning);
      const piClass = classifyInferenceSeparation(caseNum, statedMeaning, possibleInference);

      results.push({
        caseNumber: caseNum,
        case: c,
        rawAnswer: c.userAnswer,
        output: result,
        statedMeaning,
        possibleInference,
        statedMeaningClassification: smClass.classification,
        statedMeaningReasoning: smClass.reasoning,
        inferenceSeparationClassification: piClass.classification,
        inferenceSeparationReasoning: piClass.reasoning,
        timingMs: elapsed,
      });

      expect(result.statedMeaning).toBeDefined();
      expect(typeof result.statedMeaning).toBe("string");
      expect(result.statedMeaning.trim().length).toBeGreaterThan(0);
      if (result.possibleInference !== null) {
        expect(typeof result.possibleInference).toBe("string");
      }
    }, 120000);
  }

  // Aggregate analysis
  it("55D: aggregate results and semantic review", () => {
    const smCounts = { stated_meaning_preserved: 0, stated_meaning_strengthened: 0, stated_meaning_lost: 0 };
    const piCounts = { inference_cleanly_separated: 0, inference_leaked_into_stated: 0, unnecessary_inference: 0, no_inference_needed: 0 };

    for (const r of results) {
      smCounts[r.statedMeaningClassification]++;
      piCounts[r.inferenceSeparationClassification]++;
    }

    const totalMs = timings.reduce((s, t) => s + t.ms, 0);
    const msArr = timings.map((t) => t.ms);

    console.log("\n========== Experiment 55D Results ==========");
    console.log(`\nClarification target: ${CLARIFICATION_TARGET}`);
    console.log(`Question: ${CLARIFICATION_QUESTION}`);

    for (const r of results) {
      console.log(`\n--- ${r.case.id} ---`);
      console.log("Raw answer:", `"${r.rawAnswer}"`);
      console.log("statedMeaning:", `"${r.statedMeaning}"`);
      console.log("possibleInference:", r.possibleInference ?? "null");
      console.log("statedMeaning classification:", r.statedMeaningClassification);
      console.log("Reasoning:", r.statedMeaningReasoning);
      console.log("Inference separation:", r.inferenceSeparationClassification);
      console.log("Inference reasoning:", r.inferenceSeparationReasoning);
    }

    console.log("\n--- Stated Meaning Counts ---");
    Object.entries(smCounts).forEach(([k, v]) => console.log(`${k}: ${v}`));

    console.log("\n--- Inference Separation Counts ---");
    Object.entries(piCounts).forEach(([k, v]) => console.log(`${k}: ${v}`));

    console.log("\n--- Timing ---");
    console.log("Total live calls:", timings.length);
    console.log("Total time:", totalMs + "ms");
    console.log("Average:", (totalMs / timings.length).toFixed(1) + "ms per call");
    console.log("Fastest:", Math.min(...msArr) + "ms");
    console.log("Slowest:", Math.max(...msArr) + "ms");

    // Required questions
    const case1 = results.find(r => r.caseNumber === 1);
    const case2 = results.find(r => r.caseNumber === 2);
    const case3 = results.find(r => r.caseNumber === 3);
    const case4 = results.find(r => r.caseNumber === 4);

    const q1 = case1.statedMeaningClassification === "stated_meaning_preserved";
    const q2 = case1.inferenceSeparationClassification === "no_inference_needed" ||
               case1.inferenceSeparationClassification === "inference_cleanly_separated";

    console.log("\n--- Required Questions ---");
    console.log("Q1 (Case 1 kept relative importance only in statedMeaning?):", q1 ? "Yes" : "No - " + case1.statedMeaningReasoning);
    console.log("Q2 (Case 1 placed stronger implication only in possibleInference):", q2 ? "Yes" : "No - " + case1.inferenceSeparationReasoning);
    console.log("Q3 (Case 2 preserved conditionality?):", case2.statedMeaningClassification === "stated_meaning_preserved" ? "Yes" : "No - " + case2.statedMeaningReasoning);
    console.log("Q4 (Case 3 preserved explicit constraint without unnecessary inference?):",
      case3.statedMeaningClassification === "stated_meaning_preserved" && case3.inferenceSeparationClassification !== "inference_leaked_into_stated" ? "Yes" : "No" + (case3.inferenceSeparationClassification === "unnecessary_inference" ? " - unnecessary inference present" : " - check reasoning") + " - " + case3.inferenceSeparationReasoning);
    console.log("Q5 (Case 4 preserved uncertainty without inventing leaning?):", case4.statedMeaningClassification === "stated_meaning_preserved" ? "Yes" : "No - " + case4.statedMeaningReasoning);

    const anyLeaked = results.some(r => r.inferenceSeparationClassification === "inference_leaked_into_stated");
    console.log("Q6 (Did any unsupported meaning leak into statedMeaning?):", anyLeaked ? "Yes - check cases above" : "No observed leakage");

    const anyUnnecessaryInference = results.some(r => r.inferenceSeparationClassification === "unnecessary_inference");
    console.log("Q7 (Did model generate unnecessary implications where answer was explicit?):", anyUnnecessaryInference ? "Yes - check cases above" : "No unnecessary inferences observed");

    console.log("\n--- Required Answers to Critical Questions ---");
    console.log("Q10 (proves production should use this exact contract?): No");
    console.log("Q11 (establishes how resolution should consume these fields?): No");
    console.log("Q12 (establishes graph or Behaviour Selection changes?): No");

    results.summary = {
      smCounts,
      piCounts,
      totalTimeMs: totalMs,
      totalCalls: timings.length,
      averageMs: (totalMs / timings.length).toFixed(1),
      fastestMs: Math.min(...msArr),
      slowestMs: Math.max(...msArr),
      caseResults: results.map(r => ({
        caseNumber: r.caseNumber,
        caseId: r.case.id,
        rawAnswer: r.rawAnswer,
        statedMeaning: r.statedMeaning,
        possibleInference: r.possibleInference,
        statedMeaningClassification: r.statedMeaningClassification,
        inferenceSeparationClassification: r.inferenceSeparationClassification,
      })),
    };

    console.log("\n========== End of Experiment 55D ==========\n");
  });
});