experiment: test clarification null stability
This commit is contained in:
@@ -0,0 +1,272 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { config } from "dotenv";
|
||||
import path from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
config({ path: path.resolve(__dirname, "../../.env.local") });
|
||||
|
||||
const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL;
|
||||
const OLLAMA_MODEL = process.env.OLLAMA_MODEL;
|
||||
|
||||
if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) {
|
||||
throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local");
|
||||
}
|
||||
|
||||
/**
|
||||
* Make one live Ollama chat call: identify the specific user-owned
|
||||
* distinction that remains unresolved when clarification is required.
|
||||
* Uses the exact Experiment 54S instruction unchanged.
|
||||
*/
|
||||
async function callClarificationTarget(source, disagreement, requiresUserClarification) {
|
||||
const instruction = `Identify the specific unresolved distinction that only the user can clarify.
|
||||
|
||||
If clarification is required (requiresUserClarification: true), return the smallest statement of the missing user-owned meaning, preference, priority, constraint, definition, or private fact.
|
||||
|
||||
If clarification is not required (requiresUserClarification: false), return null.
|
||||
|
||||
Do not write a question. Do not add evidence needs. Do not select a preferred interpretation.
|
||||
|
||||
Return valid JSON only in this shape:
|
||||
{
|
||||
"clarificationTarget": "short statement" | null
|
||||
}`;
|
||||
|
||||
const messages = [
|
||||
{ role: "system", content: instruction.trim() },
|
||||
{
|
||||
role: "user",
|
||||
content: `Source: ${JSON.stringify(source)}
|
||||
|
||||
Disagreement:
|
||||
${disagreement.map((d, i) => `${i + 1}. ${d}`).join("\n")}
|
||||
|
||||
requiresUserClarification: ${requiresUserClarification}`,
|
||||
},
|
||||
];
|
||||
|
||||
const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: OLLAMA_MODEL,
|
||||
messages,
|
||||
format: "json",
|
||||
stream: false,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
const rawContent = data.message?.content ?? "";
|
||||
const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");
|
||||
|
||||
return JSON.parse(cleaned.trim());
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Human reference for Case B semantic review
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
const HUMAN_REF_CASE_B = {
|
||||
expectedDistinction:
|
||||
"whether avoiding additional risk is a preference/trade-off or a hard constraint",
|
||||
requiredConcepts: [
|
||||
"risk",
|
||||
"constraint",
|
||||
"preference",
|
||||
"trade-off",
|
||||
"avoiding",
|
||||
"additional",
|
||||
"hard",
|
||||
],
|
||||
forbiddenConcepts: ["evidence", "investigate", "check", "look at", "data"],
|
||||
};
|
||||
|
||||
function evaluateCaseBTarget(target) {
|
||||
if (target == null) return { classification: "null", target };
|
||||
if (typeof target !== "string" || !target.trim())
|
||||
return { classification: "null", target };
|
||||
|
||||
const t = target.trim().toLowerCase();
|
||||
if (t.endsWith("?")) return { classification: "null", target };
|
||||
|
||||
// Semantic check against human reference
|
||||
const reqConcepts = HUMAN_REF_CASE_B.requiredConcepts;
|
||||
const hasRequired = reqConcepts.some(
|
||||
(c) => t.includes(c.toLowerCase())
|
||||
);
|
||||
|
||||
const forbiddenConcepts = HUMAN_REF_CASE_B.forbiddenConcepts;
|
||||
const hasForbidden = forbiddenConcepts.some(
|
||||
(c) => t.includes(c.toLowerCase())
|
||||
);
|
||||
|
||||
if (!hasRequired || hasForbidden) {
|
||||
return { classification: "target_incorrect", target };
|
||||
}
|
||||
|
||||
return { classification: "target_correct", target };
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Fixed cases
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
const CASE_A = {
|
||||
id: "Case A — Evidence-Resolvable / False",
|
||||
source: "Orders are arriving late and customers have started complaining.",
|
||||
disagreement: [
|
||||
"delays may be caused by insufficient staff capacity",
|
||||
"delays may be caused by unreliable supplier lead times",
|
||||
],
|
||||
requiresUserClarification: false,
|
||||
expectedNull: true,
|
||||
};
|
||||
|
||||
const CASE_B = {
|
||||
id: "Case B — User-Owned Ambiguity / True Control",
|
||||
source: "I want the business to grow, but I don't want to take on more risk.",
|
||||
disagreement: [
|
||||
"growth should be prioritised even if some additional risk is unavoidable",
|
||||
"avoiding additional risk is a hard constraint even if growth is slower",
|
||||
],
|
||||
requiresUserClarification: true,
|
||||
expectedNull: false,
|
||||
};
|
||||
|
||||
const RUNS_PER_CASE = 3;
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Test suite — Case A (false → null stability)
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("Experiment 54T — Clarification Null Stability", () => {
|
||||
const results = { a: [], b: [] };
|
||||
const timings = [];
|
||||
|
||||
describe("Case A — Evidence-Resolvable / False (repeated " + RUNS_PER_CASE + "×)", () => {
|
||||
for (let i = 0; i < RUNS_PER_CASE; i++) {
|
||||
it(`run ${i + 1}`, async () => {
|
||||
const start = Date.now();
|
||||
const result = await callClarificationTarget(
|
||||
CASE_A.source,
|
||||
CASE_A.disagreement,
|
||||
CASE_A.requiresUserClarification
|
||||
);
|
||||
const elapsed = Date.now() - start;
|
||||
timings.push({ caseId: CASE_A.id, run: i + 1, ms: elapsed });
|
||||
|
||||
const isNull = result.clarificationTarget == null;
|
||||
results.a.push({ run: i + 1, result, isNull });
|
||||
|
||||
// Structural check
|
||||
expect(result.clarificationTarget).toBeDefined();
|
||||
|
||||
// Null-gating: false → null expected
|
||||
if (CASE_A.expectedNull) {
|
||||
console.log(
|
||||
`[54T Case A run ${i + 1}] clarificationTarget: ${isNull ? "null ✓" : `"${result.clarificationTarget}" ✗`}`
|
||||
);
|
||||
expect(isNull).toBe(true);
|
||||
} else {
|
||||
expect(isNull).toBe(false);
|
||||
}
|
||||
}, 120000);
|
||||
}
|
||||
});
|
||||
|
||||
describe("Case B — User-Owned Ambiguity / True Control (repeated " + RUNS_PER_CASE + "×)", () => {
|
||||
for (let i = 0; i < RUNS_PER_CASE; i++) {
|
||||
it(`run ${i + 1}`, async () => {
|
||||
const start = Date.now();
|
||||
const result = await callClarificationTarget(
|
||||
CASE_B.source,
|
||||
CASE_B.disagreement,
|
||||
CASE_B.requiresUserClarification
|
||||
);
|
||||
const elapsed = Date.now() - start;
|
||||
timings.push({ caseId: CASE_B.id, run: i + 1, ms: elapsed });
|
||||
|
||||
const ev = evaluateCaseBTarget(result.clarificationTarget);
|
||||
results.b.push({ run: i + 1, result, classification: ev.classification });
|
||||
|
||||
// Structural check
|
||||
expect(result.clarificationTarget).toBeDefined();
|
||||
|
||||
console.log(
|
||||
`[54T Case B run ${i + 1}] classification: ${ev.classification} | target: "${result.clarificationTarget ?? "null"}"`
|
||||
);
|
||||
|
||||
// true → non-null expected
|
||||
if (CASE_B.expectedNull) {
|
||||
expect(ev.classification).not.toBe("null");
|
||||
} else {
|
||||
// This case expects a non-null correct target
|
||||
expect(ev.classification).toBe("target_correct");
|
||||
}
|
||||
}, 120000);
|
||||
}
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Aggregate summary
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
it("54T aggregate results", () => {
|
||||
const aNull = results.a.filter((r) => r.isNull).length;
|
||||
const aNonNull = results.a.filter((r) => !r.isNull).length;
|
||||
|
||||
const bCorrect = results.b.filter((r) => r.classification === "target_correct").length;
|
||||
const bIncorrect = results.b.filter((r) => r.classification === "target_incorrect").length;
|
||||
const bNull = results.b.filter((r) => r.classification === "null").length;
|
||||
|
||||
const totalMs = timings.reduce((s, t) => s + t.ms, 0);
|
||||
const avgMs = totalMs / timings.length;
|
||||
const fastMs = Math.min(...timings.map((t) => t.ms));
|
||||
const slowMs = Math.max(...timings.map((t) => t.ms));
|
||||
|
||||
console.log("\n=== Experiment 54T Aggregate Results ===");
|
||||
console.log(`\n--- Case A (false → null) ---`);
|
||||
for (const r of results.a) {
|
||||
console.log(
|
||||
`Run ${r.run}: ${r.isNull ? "null" : `"${r.result.clarificationTarget}"`}`
|
||||
);
|
||||
}
|
||||
console.log(`Null: ${aNull}/${results.a.length}`);
|
||||
console.log(`Non-null: ${aNonNull}/${results.a.length}`);
|
||||
|
||||
if (aNonNull > 0) {
|
||||
console.log("\nInvented clarification targets:");
|
||||
results
|
||||
.filter((r) => !r.isNull)
|
||||
.forEach((r) => console.log(` - "${r.result.clarificationTarget}"`));
|
||||
}
|
||||
|
||||
console.log(`\n--- Case B (true → target) ---`);
|
||||
for (const r of results.b) {
|
||||
console.log(
|
||||
`Run ${r.run}: ${r.classification} | "${r.result ?? "null"}"`
|
||||
);
|
||||
}
|
||||
console.log(`Correct: ${bCorrect}/${results.b.length}`);
|
||||
console.log(`Incorrect: ${bIncorrect}/${results.b.length}`);
|
||||
console.log(`Null: ${bNull}/${results.b.length}`);
|
||||
|
||||
console.log(`\n--- Timing ---`);
|
||||
console.log(`Total live calls: ${timings.length}`);
|
||||
console.log(`Total time: ${totalMs}ms (${(totalMs / 1000).toFixed(1)}s)`);
|
||||
console.log(`Average: ${avgMs.toFixed(1)}ms per call`);
|
||||
console.log(`Fastest: ${fastMs}ms`);
|
||||
console.log(`Slowest: ${slowMs}ms`);
|
||||
|
||||
// Summary assertions
|
||||
expect(timings.length).toBe(6);
|
||||
expect(aNull + aNonNull).toBe(3);
|
||||
expect(bCorrect + bIncorrect + bNull).toBe(3);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user