import { describe, it, expect } from "vitest"; import { config } from "dotenv"; import path from "path"; import { fileURLToPath } from "url"; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); config({ path: path.resolve(__dirname, "../../.env.local") }); const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL; const OLLAMA_MODEL = process.env.OLLAMA_MODEL; if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) { throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local"); } // Regression D — Explicit hard constraint const SYSTEM_INSTRUCTION = `You are evaluating whether a user's answer preserves explicit meaning without weakening it. State only what the user's answer directly establishes in userSupportedMeaning. Preserve qualification and absoluteness. Do not turn a hard boundary into a preference or trade-off unless the user actually says so. If there is a plausible implication that goes beyond what the answer directly establishes, place it only in possibleInference (string or null). Do not decide whether clarification is resolved. Do not recommend action.`; const CLARIFICATION_TARGET = "whether avoiding additional risk is a hard constraint or a preference/trade-off"; const CLARIFICATION_QUESTION = "Do you view avoiding additional risk as a hard constraint, or as a preference or trade-off?"; const USER_ANSWER = "It's a hard constraint. I don't want any increase in risk."; const HUMAN_REFERENCE = "Avoiding additional risk is an explicit hard constraint. The user does not accept any increase in risk."; async function callRegressionD() { const messages = [ { role: "system", content: SYSTEM_INSTRUCTION.trim() }, { role: "user", content: `Clarification target context: ${CLARIFICATION_TARGET}\n\nClarification question: ${CLARIFICATION_QUESTION}\n\nUser's answer: ${USER_ANSWER}` }, ]; const start = Date.now(); const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model: OLLAMA_MODEL, messages, format: "json", stream: false }), }); const duration = Date.now() - start; if (!res.ok) throw new Error(`Ollama API error: ${res.status} ${res.statusText}`); const data = await res.json(); const rawContent = data.message?.content ?? ""; const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, ""); return { parsed: JSON.parse(cleaned.trim()), duration }; } describe("Experiment 56J — Explicit Hard Constraint Semantic Fidelity (Regression D)", () => { it("Regression D: one live call preserves hard-constraint meaning", async () => { const result = await callRegressionD(); const { parsed, duration } = result; const userSupportedMeaning = parsed.userSupportedMeaning ?? ""; const possibleInference = parsed.possibleInference ?? null; expect(userSupportedMeaning).toBeDefined(); expect(typeof userSupportedMeaning).toBe("string"); expect(userSupportedMeaning.trim().length).toBeGreaterThan(0); if (possibleInference !== null) expect(typeof possibleInference).toBe("string"); console.log("\n========== Experiment 56J — Regression D =========="); console.log(`Configured model: ${OLLAMA_MODEL}`); console.log(`Configured base URL: ${OLLAMA_BASE_URL}`); console.log(`Call count: 1`); console.log(`Duration: ${duration}ms`); console.log(`\n--- Fixed Input ---`); console.log(`Clarification target: ${CLARIFICATION_TARGET}`); console.log(`Question: ${CLARIFICATION_QUESTION}`); console.log(`User answer: "${USER_ANSWER}"`); console.log(`\n--- Raw Parsed Response ---`); console.log(JSON.stringify(parsed, null, 2)); console.log(`\n--- userSupportedMeaning ---`); console.log(userSupportedMeaning); console.log(`\n--- possibleInference ---`); console.log(possibleInference ?? "null"); console.log(`\n--- Pre-written Human Reference ---`); console.log(HUMAN_REFERENCE); const sm = userSupportedMeaning.toLowerCase().trim(); const piText = typeof possibleInference === "string" ? possibleInference.toLowerCase().trim() : ""; const hasHardConstraint = /hard.*constraint|definitively.*not|absolute.*boundary|non.?negotiable|no.*acceptable|won't.*accept.*any|must.*avoid.*any/i.test(sm); const weakenedToPreference = /^(prefers?|strong.*preference|tends to|would like|should|concerned about|matters more|important|desires)/i.test(sm); let classification; let rationale; if (hasHardConstraint) { if (sm.includes("uncertain") || sm.includes("don't know") || sm.includes("not sure")) { classification = "FAIL"; rationale = `Explicit hard constraint replaced with uncertainty: "${userSupportedMeaning}"`; } else if (weakenedToPreference) { classification = "FAIL"; rationale = `Hard constraint meaning weakened into preference/trade-off language: "${userSupportedMeaning}"`; } else { if (possibleInference !== null && possibleInference.trim().length > 0) { classification = "PASS"; rationale = `Hard constraint preserved clearly. possibleInference present (${possibleInference}) but userSupportedMeaning is clean and unweakened.`; } else { classification = "PASS"; rationale = `Hard constraint preserved clearly with no unnecessary inference: "${userSupportedMeaning}"`; } } } else if (!/uncertain|don't.*know|not.*sure|unsure/i.test(sm)) { if (weakenedToPreference) { classification = "FAIL"; rationale = `Hard constraint weakened into preference/trade-off language: "${userSupportedMeaning}"`; } else { classification = "UNRESOLVED"; rationale = `Output does not establish enough meaning to judge faithfully.`; } } else { classification = "UNRESOLVED"; rationale = `Output replaced explicit meaning with uncertainty: "${userSupportedMeaning}"`; } console.log(`\n--- Human Semantic Classification ---`); console.log(classification); console.log(rationale); result._classification = classification; result._rationale = rationale; expect(classification).not.toBe("FAIL"); }, 120000); });