confidence-engine/tests/reconstruction/semantic-regression-d-explicit-hard-constraint.test.js

import { describe, it, expect } from "vitest";
import { config } from "dotenv";
import path from "path";
import { fileURLToPath } from "url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
config({ path: path.resolve(__dirname, "../../.env.local") });

const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL;
const OLLAMA_MODEL = process.env.OLLAMA_MODEL;

if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) {
  throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local");
}

// Regression D — Explicit hard constraint
const SYSTEM_INSTRUCTION = `You are evaluating whether a user's answer preserves explicit meaning without weakening it.

State only what the user's answer directly establishes in userSupportedMeaning. Preserve qualification and absoluteness. Do not turn a hard boundary into a preference or trade-off unless the user actually says so.

If there is a plausible implication that goes beyond what the answer directly establishes, place it only in possibleInference (string or null).

Do not decide whether clarification is resolved. Do not recommend action.`;

const CLARIFICATION_TARGET = "whether avoiding additional risk is a hard constraint or a preference/trade-off";
const CLARIFICATION_QUESTION = "Do you view avoiding additional risk as a hard constraint, or as a preference or trade-off?";
const USER_ANSWER = "It's a hard constraint. I don't want any increase in risk.";

const HUMAN_REFERENCE = "Avoiding additional risk is an explicit hard constraint. The user does not accept any increase in risk.";

async function callRegressionD() {
  const messages = [
    { role: "system", content: SYSTEM_INSTRUCTION.trim() },
    { role: "user", content: `Clarification target context: ${CLARIFICATION_TARGET}\n\nClarification question: ${CLARIFICATION_QUESTION}\n\nUser's answer: ${USER_ANSWER}` },
  ];

  const start = Date.now();
  const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ model: OLLAMA_MODEL, messages, format: "json", stream: false }),
  });
  const duration = Date.now() - start;

  if (!res.ok) throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);

  const data = await res.json();
  const rawContent = data.message?.content ?? "";
  const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");

  return { parsed: JSON.parse(cleaned.trim()), duration };
}

describe("Experiment 56J — Explicit Hard Constraint Semantic Fidelity (Regression D)", () => {
  it("Regression D: one live call preserves hard-constraint meaning", async () => {
    const result = await callRegressionD();
    const { parsed, duration } = result;
    const userSupportedMeaning = parsed.userSupportedMeaning ?? "";
    const possibleInference = parsed.possibleInference ?? null;

    expect(userSupportedMeaning).toBeDefined();
    expect(typeof userSupportedMeaning).toBe("string");
    expect(userSupportedMeaning.trim().length).toBeGreaterThan(0);
    if (possibleInference !== null) expect(typeof possibleInference).toBe("string");

    console.log("\n========== Experiment 56J — Regression D ==========");
    console.log(`Configured model: ${OLLAMA_MODEL}`);
    console.log(`Configured base URL: ${OLLAMA_BASE_URL}`);
    console.log(`Call count: 1`);
    console.log(`Duration: ${duration}ms`);
    console.log(`\n--- Fixed Input ---`);
    console.log(`Clarification target: ${CLARIFICATION_TARGET}`);
    console.log(`Question: ${CLARIFICATION_QUESTION}`);
    console.log(`User answer: "${USER_ANSWER}"`);
    console.log(`\n--- Raw Parsed Response ---`);
    console.log(JSON.stringify(parsed, null, 2));
    console.log(`\n--- userSupportedMeaning ---`);
    console.log(userSupportedMeaning);
    console.log(`\n--- possibleInference ---`);
    console.log(possibleInference ?? "null");
    console.log(`\n--- Pre-written Human Reference ---`);
    console.log(HUMAN_REFERENCE);

    const sm = userSupportedMeaning.toLowerCase().trim();
    const piText = typeof possibleInference === "string" ? possibleInference.toLowerCase().trim() : "";
    const hasHardConstraint = /hard.*constraint|definitively.*not|absolute.*boundary|non.?negotiable|no.*acceptable|won't.*accept.*any|must.*avoid.*any/i.test(sm);
    const weakenedToPreference = /^(prefers?|strong.*preference|tends to|would like|should|concerned about|matters more|important|desires)/i.test(sm);

    let classification;
    let rationale;

    if (hasHardConstraint) {
      if (sm.includes("uncertain") || sm.includes("don't know") || sm.includes("not sure")) {
        classification = "FAIL";
        rationale = `Explicit hard constraint replaced with uncertainty: "${userSupportedMeaning}"`;
      } else if (weakenedToPreference) {
        classification = "FAIL";
        rationale = `Hard constraint meaning weakened into preference/trade-off language: "${userSupportedMeaning}"`;
      } else {
        if (possibleInference !== null && possibleInference.trim().length > 0) {
          classification = "PASS";
          rationale = `Hard constraint preserved clearly. possibleInference present (${possibleInference}) but userSupportedMeaning is clean and unweakened.`;
        } else {
          classification = "PASS";
          rationale = `Hard constraint preserved clearly with no unnecessary inference: "${userSupportedMeaning}"`;
        }
      }
    } else if (!/uncertain|don't.*know|not.*sure|unsure/i.test(sm)) {
      if (weakenedToPreference) {
        classification = "FAIL";
        rationale = `Hard constraint weakened into preference/trade-off language: "${userSupportedMeaning}"`;
      } else {
        classification = "UNRESOLVED";
        rationale = `Output does not establish enough meaning to judge faithfully.`;
      }
    } else {
      classification = "UNRESOLVED";
      rationale = `Output replaced explicit meaning with uncertainty: "${userSupportedMeaning}"`;
    }

    console.log(`\n--- Human Semantic Classification ---`);
    console.log(classification);
    console.log(rationale);

    result._classification = classification;
    result._rationale = rationale;

    expect(classification).not.toBe("FAIL");
  }, 120000);
});