experiment: validate grounded unclassified answer live

2026-08-09 20:03:30 +01:00
parent 4e4d0fa732
commit 19a42ca7f7
3 changed files with 175 additions and 0 deletions
@@ -0,0 +1,102 @@
+import { describe, it, expect } from "vitest";
+import { config } from "dotenv";
+import path from "path";
+import { fileURLToPath } from "url";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+config({ path: path.resolve(__dirname, "../../.env.local") });
+
+const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL;
+const OLLAMA_MODEL = process.env.OLLAMA_MODEL;
+
+if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) {
+  throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local");
+}
+
+// Fixed case from Experiment 57A — the affirmative answer that exposed the defect
+const UNRESOLVED_QUESTION = "Whether cost reduction is a genuine reason supporting the relocation decision.";
+const USER_ANSWER = "We're looking at this mainly for cost reduction — roughly £2M annual savings on office overhead.";
+
+// Pre-written human semantic reference (authoritative)
+const HUMAN_REFERENCE = `The answer establishes that cost reduction is a genuine stated reason supporting consideration of the relocation, with approximately £2M annual office-overhead savings cited by the user. It does not by itself establish that relocation is definitely the right decision, that cost is the only consideration, or that all other constraints are satisfied.`;
+
+// Live call
+async function callUnclassifiedAffirmative() {
+  const instruction = `You are validating whether a model-produced interpretation of a raw user answer stays grounded in what was actually stated.
+
+Pre-written human semantic reference (authoritative): ${HUMAN_REFERENCE}
+
+Context / unresolved question: ${UNRESOLVED_QUESTION}
+
+User's answer: "${USER_ANSWER}"
+
+Return valid JSON only in this shape:
+{
+  "userSupportedMeaning": "short statement of what the user actually established",
+  "possibleInference": "short statement or null"
+}
+
+Rules:
+- userSupportedMeaning must stay within what the raw answer directly establishes.
+- possibleInference captures a plausible implication that goes beyond the raw answer (or null).
+- Do not strengthen the meaning into a final decision, hard constraint, preference judgment, or approval.`;
+
+  const messages = [
+    { role: "system", content: instruction.trim() },
+    { role: "user", content: USER_ANSWER },
+  ];
+
+  const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      model: OLLAMA_MODEL,
+      messages,
+      format: "json",
+      stream: false,
+    }),
+  });
+
+  if (!res.ok) {
+    throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
+  }
+
+  const data = await res.json();
+  const rawContent = data.message?.content ?? "";
+  const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");
+
+  return JSON.parse(cleaned.trim());
+}
+
+// Test suite — single live call, Experiment 57B
+describe("Experiment 57B - Grounded unclassified affirmative answer", () => {
+  let result;
+  let durationMs;
+
+  it("57B: one live call — unclassified affirmative answer stays grounded", async () => {
+    const start = Date.now();
+    result = await callUnclassifiedAffirmative();
+    durationMs = Date.now() - start;
+
+    // Minimal structural assertions
+    expect(result.userSupportedMeaning).toBeDefined();
+    expect(typeof result.userSupportedMeaning).toBe("string");
+    expect(result.userSupportedMeaning.trim().length).toBeGreaterThan(0);
+    if (result.possibleInference !== null) {
+      expect(typeof result.possibleInference).toBe("string");
+    }
+  }, 120000);
+
+  it("57B: raw result report", () => {
+    console.log("\n========== Experiment 57B Results ==========");
+    console.log(`\nContext / unresolved question: ${UNRESOLVED_QUESTION}`);
+    console.log(`User answer: "${USER_ANSWER}"`);
+    console.log(`Pre-written human reference:\n${HUMAN_REFERENCE}`);
+    console.log(`\nRaw structured response:`);
+    console.log(`  userSupportedMeaning: "${result?.userSupportedMeaning}"`);
+    console.log(`  possibleInference: ${result?.possibleInference ?? "null"}`);
+    console.log(`\nDuration: ${durationMs}ms`);
+    console.log("\n========== End of Experiment 57B ==========\n");
+  });
+});