experiment: probe user-owned ambiguity boundary

2026-08-09 15:21:02 +01:00
parent 11882bfaae
commit e884b02e7c
3 changed files with 308 additions and 1 deletions
@@ -0,0 +1,217 @@
+import { describe, it, expect } from "vitest";
+import { config } from "dotenv";
+import path from "path";
+import { fileURLToPath } from "url";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+config({ path: path.resolve(__dirname, "../../.env.local") });
+
+const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL;
+const OLLAMA_MODEL = process.env.OLLAMA_MODEL;
+
+if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) {
+  throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local");
+}
+
+// ── Regression F — User-owned ambiguity ──────────────────────────────────────
+
+const REGRESSION_F_CASE = {
+  id: "Regression F - User-owned ambiguity",
+  source: "I want the business to grow, but I don't want to take on more risk.",
+  answer: "(ambiguous statement about both growth and risk — user has not specified preference/trade-off versus hard constraint)",
+  expectedPreservedMeaning: "User has not specified whether avoiding additional risk is a hard constraint or a strong preference/trade-off.",
+  expectedUncertainty: "Preference vs constraint distinction is user-owned and requires clarification.",
+  mustNotHappen: 'Engine-generated classification of the ambiguity as "not requiring clarification" or resolution through evidence gathering alone.',
+};
+
+// Pre-written human reference — derived from Regression F requirement, written before model output
+const HUMAN_REFERENCE = {
+  uncertaintyType: "user_clarification_needed",
+  reason: "The unresolved distinction (whether avoiding additional risk is a hard constraint or a strong preference/trade-off) belongs to the user's own meaning. External evidence cannot establish what the user means, prefers, intends, defines, or constrains on their own behalf.",
+  evidenceNeeded: null,
+  userClarificationNeeded: "Whether the user treats avoiding additional risk as a hard constraint (non-negotiable boundary) versus a strong preference/trade-off (weighted but potentially overrideable).",
+};
+
+// ── Live call ────────────────────────────────────────────────────────────────
+
+async function probeUserOwnedAmbiguity() {
+  const instruction = `You are evaluating an ambiguity in user decision-making. The distinction is between two categories:
+
+- "evidence_needed": the uncertainty can be resolved by gathering external facts or data.
+- "user_clarification_needed": only the user can establish what they mean, prefer, intend, define, or constrain. External evidence cannot determine their private meaning.
+
+Read the case below and classify which category applies. Return ONLY valid JSON matching exactly this schema:
+
+{
+  "uncertaintyType": "evidence_needed | user_clarification_needed | unresolved",
+  "reason": "brief explanation of your classification",
+  "evidenceNeeded": "string describing what external evidence would resolve this, or null if not applicable",
+  "userClarificationNeeded": "string describing what only the user can establish, or null if not applicable"
+}
+
+Do not add fields. Do not add prose outside the JSON.`;
+
+  const messages = [
+    { role: "system", content: instruction.trim() },
+    {
+      role: "user",
+      content: `Case: Regression F - User-owned ambiguity
+
+Source: "${REGRESSION_F_CASE.source}"
+
+Answer context: ${REGRESSION_F_CASE.answer}
+
+Expected preserved meaning: ${REGRESSION_F_CASE.expectedPreservedMeaning}
+
+Expected uncertainty: ${REGRESSION_F_CASE.expectedUncertainty}
+
+Must not happen: ${REGRESSION_F_CASE.mustNotHappen}
+
+Classify the type of uncertainty and return JSON.`,
+    },
+  ];
+
+  const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      model: OLLAMA_MODEL,
+      messages,
+      format: "json",
+      stream: false,
+    }),
+  });
+
+  if (!res.ok) {
+    throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
+  }
+
+  const data = await res.json();
+  const rawContent = data.message?.content ?? "";
+  const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");
+
+  return JSON.parse(cleaned.trim());
+}
+
+// ── Human semantic classification helpers (minimal) ─────────────────────────
+
+function classifyResult(result) {
+  const ut = result.uncertaintyType ?? "";
+  const reason = (result.reason ?? "").toLowerCase().trim();
+  const ev = result.evidenceNeeded;
+  const uc = result.userClarificationNeeded;
+
+  const identifiesUserOwned =
+    ut === "user_clarification_needed" &&
+    !/evidence.*can.*establish|external.*facts.*determine|can.*be.*resolved.*by.*evidence|does not require clarification/i.test(reason);
+
+  const treatsAsEvidence =
+    ut === "evidence_needed" ||
+    /only evidence|requires only evidence|external facts can determine|can be resolved by gathering|does not need clarification|not user-owned/i.test(reason) ||
+    (ut === "unresolved" && !reason.match(/clarification|preference|constraint|intention|meaning|user.*mean/i));
+
+  if (identifiesUserOwned) {
+    const hasEvidenceBleed = ev !== null && typeof ev === "string" && ev.trim().length > 0;
+    const hasBlurryLanguage = /both|may require.*evidence.*and.*clarification|depends on.*which/i.test(reason);
+
+    if (hasEvidenceBleed && !hasBlurryLanguage) {
+      return { classification: "PARTIAL", rationale: `Model identified user_clarification_needed but also suggested evidence gathering (${ev}). The distinction is recognized but not cleanly preserved.` };
+    }
+    if (hasBlurryLanguage) {
+      return { classification: "PARTIAL", rationale: `Model identified user clarification but language blurs the evidence vs clarification boundary: "${reason}"` };
+    }
+    return { classification: "PASS", rationale: `Model clearly recognises the ambiguity belongs to the user's own meaning and requires user clarification rather than external evidence. Preserved the distinction cleanly.` };
+  }
+
+  if (treatsAsEvidence) {
+    return { classification: "FAIL", rationale: `Model treats user-owned ambiguity as an evidence-gathering problem. uncertaintyType="${ut}", reason="${reason}"` };
+  }
+
+  return { classification: "UNRESOLVED", rationale: `Model avoided unjustified classification. uncertaintyType="${ut}", reason="${reason}"` };
+}
+
+// ── Test suite ───────────────────────────────────────────────────────────────
+
+describe("Experiment 56L - User-owned ambiguity requires clarification", () => {
+  it(
+    "Regression F: one live call to probe user-owned ambiguity boundary",
+    async () => {
+      const start = Date.now();
+      const result = await probeUserOwnedAmbiguity();
+      const elapsedMs = Date.now() - start;
+
+      expect(result.uncertaintyType).toBeDefined();
+      expect(["evidence_needed", "user_clarification_needed", "unresolved"]).toContain(result.uncertaintyType);
+      expect(typeof result.reason).toBe("string");
+      expect(result.reason.length).toBeGreaterThan(0);
+      expect(result.evidenceNeeded === null || typeof result.evidenceNeeded === "string").toBe(true);
+      expect(result.userClarificationNeeded === null || typeof result.userClarificationNeeded === "string").toBe(true);
+
+      globalThis._exp56lResult = result;
+      globalThis._exp56lElapsedMs = elapsedMs;
+    },
+    300000
+  );
+
+  it("56L: human semantic classification", () => {
+    const result = globalThis._exp56lResult;
+    const elapsedMs = globalThis._exp56lElapsedMs;
+
+    if (!result) {
+      throw new Error("Live call must run first — did it time out?");
+    }
+
+    const classResult = classifyResult(result);
+
+    console.log("\n========== Experiment 56L Results ==========");
+    console.log(`\n--- Config ---`);
+    console.log(`Ollama base URL: ${OLLAMA_BASE_URL}`);
+    console.log(`Ollama model: ${OLLAMA_MODEL}`);
+    console.log(`Live-call count: 1`);
+    console.log(`Call duration: ${elapsedMs} ms`);
+
+    console.log(`\n--- Regression F Fixed Case ---`);
+    console.log(`Source: "${REGRESSION_F_CASE.source}"`);
+    console.log(`Answer context: ${REGRESSION_F_CASE.answer}`);
+    console.log(`Expected preserved meaning: ${REGRESSION_F_CASE.expectedPreservedMeaning}`);
+    console.log(`Expected uncertainty: ${REGRESSION_F_CASE.expectedUncertainty}`);
+    console.log(`Must not happen: ${REGRESSION_F_CASE.mustNotHappen}`);
+
+    console.log(`\n--- Pre-written Human Reference ---`);
+    console.log(`uncertaintyType: ${HUMAN_REFERENCE.uncertaintyType}`);
+    console.log(`reason: ${HUMAN_REFERENCE.reason}`);
+    console.log(`evidenceNeeded: ${HUMAN_REFERENCE.evidenceNeeded ?? "null"}`);
+    console.log(`userClarificationNeeded: ${HUMAN_REFERENCE.userClarificationNeeded}`);
+
+    console.log(`\n--- Raw Structured Response ---`);
+    console.log(`uncertaintyType: "${result.uncertaintyType}"`);
+    console.log(`reason: "${result.reason}"`);
+    console.log(`evidenceNeeded: ${result.evidenceNeeded ?? "null"}`);
+    console.log(`userClarificationNeeded: ${result.userClarificationNeeded ?? "null"}`);
+
+    console.log(`\n--- Human Semantic Classification ---`);
+    console.log(`Classification: ${classResult.classification}`);
+    console.log(`Rationale: ${classResult.rationale}`);
+
+    console.log(`\n--- Detailed Analysis ---`);
+    const recognisedUserCanResolve = result.uncertaintyType === "user_clarification_needed";
+    console.log(`Did the model recognise that only the user can resolve the ambiguity: ${recognisedUserCanResolve ? "YES" : "NO/PARTIAL"}`);
+
+    const reasonLower = (result.reason ?? "").toLowerCase();
+    const treatsAsEvidenceProb = /only evidence|requires only evidence|external.*can establish|does not require clarification/i.test(reasonLower);
+    console.log(`Did it incorrectly treat the ambiguity as an evidence problem: ${treatsAsEvidenceProb ? "YES" : "NO/PARTIAL"}`);
+
+    const preservesDistinction = result.uncertaintyType === "user_clarification_needed" && result.userClarificationNeeded !== null;
+    console.log(`Did it preserve the evidence-vs-user-meaning distinction: ${preservesDistinction ? "YES" : "NO/PARTIAL"}`);
+
+    const matchesExpected = result.uncertaintyType === HUMAN_REFERENCE.uncertaintyType;
+    console.log(`\n--- Comparison with Pre-written Human Reference ---`);
+    console.log(`Expected: ${HUMAN_REFERENCE.uncertaintyType}`);
+    console.log(`Actual:   ${result.uncertaintyType}`);
+    console.log(`Matches:  ${matchesExpected ? "YES" : "NO"}`);
+
+    // Automated checks — structural only; human classification is authoritative
+    expect(result.uncertaintyType).toBeDefined();
+  });
+});