experiment: test consequence of clarification target broadening
This commit is contained in:
@@ -0,0 +1,374 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { config } from "dotenv";
|
||||
import path from "path";
|
||||
import { fileURLToPath } from "url";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
config({ path: path.resolve(__dirname, "../../.env.local") });
|
||||
|
||||
const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL;
|
||||
const OLLAMA_MODEL = process.env.OLLAMA_MODEL;
|
||||
|
||||
if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) {
|
||||
throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local");
|
||||
}
|
||||
|
||||
/**
|
||||
* Make one live Ollama chat call: turn a fixed clarification target
|
||||
* into one concise, neutral user-facing question.
|
||||
*/
|
||||
async function callClarificationQuestion(source, clarificationTarget) {
|
||||
const instruction = `Write one concise clarification question that asks only about the supplied clarification target. Keep it neutral between the possible meanings. Do not introduce new facts, assumptions, evidence requests, recommendations, or additional questions. Do not explain why you are asking.
|
||||
|
||||
Return valid JSON only in this shape:
|
||||
{
|
||||
"question": "one clarification question"
|
||||
}`;
|
||||
|
||||
const messages = [
|
||||
{ role: "system", content: instruction.trim() },
|
||||
{
|
||||
role: "user",
|
||||
content: `Source: ${JSON.stringify(source)}
|
||||
|
||||
Clarification target: ${clarificationTarget}`,
|
||||
},
|
||||
];
|
||||
|
||||
const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: OLLAMA_MODEL,
|
||||
messages,
|
||||
format: "json",
|
||||
stream: false,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
const rawContent = data.message?.content ?? "";
|
||||
const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");
|
||||
|
||||
return JSON.parse(cleaned.trim());
|
||||
}
|
||||
|
||||
/**
|
||||
* Make one live Ollama chat call: take a clarification answer
|
||||
* and return what was resolved, whether the target is resolved,
|
||||
* and any remaining uncertainty about that specific target.
|
||||
*/
|
||||
async function callClarificationAnswerResolution(source, clarificationTarget, clarificationQuestion, userAnswer) {
|
||||
const instruction = `Use the user's clarification answer only to resolve the supplied clarification target. State the meaning now established by that answer. Mark targetResolved true only when the answer settles the target. Put any uncertainty that remains specifically about that target into remainingUncertainty; otherwise return null. Do not infer wider consequences, rewrite unrelated source meaning, recommend action, or generate another question.
|
||||
|
||||
Return valid JSON only in this shape:
|
||||
{
|
||||
"resolvedMeaning": "short statement",
|
||||
"targetResolved": true,
|
||||
"remainingUncertainty": null
|
||||
}`;
|
||||
|
||||
const messages = [
|
||||
{ role: "system", content: instruction.trim() },
|
||||
{
|
||||
role: "user",
|
||||
content: `Source: ${JSON.stringify(source)}
|
||||
|
||||
Clarification target: ${clarificationTarget}
|
||||
|
||||
Clarification question: ${clarificationQuestion}
|
||||
|
||||
User's answer: ${userAnswer}`,
|
||||
},
|
||||
];
|
||||
|
||||
const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: OLLAMA_MODEL,
|
||||
messages,
|
||||
format: "json",
|
||||
stream: false,
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
const rawContent = data.message?.content ?? "";
|
||||
const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");
|
||||
|
||||
return JSON.parse(cleaned.trim());
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Shared scenario and fixed inputs
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
const SOURCE = "I want the business to grow, but I don't want to take on more risk.";
|
||||
const FIXED_USER_ANSWER = "It's a hard constraint. I don't want any increase in risk.";
|
||||
|
||||
// Variant A -- Precise target
|
||||
const VARIANT_A_TARGET = "whether avoiding additional risk is a preference/trade-off or a hard constraint";
|
||||
|
||||
// Variant B -- Broadened target
|
||||
const VARIANT_B_TARGET = "priority between business growth and risk avoidance when they conflict";
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Semantic helpers
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Determine whether two questions ask the user to resolve the same
|
||||
* underlying conceptual distinction (not just different wording).
|
||||
*/
|
||||
function classifyQuestionEquivalence(qA, qB) {
|
||||
const a = qA.toLowerCase();
|
||||
const b = qB.toLowerCase();
|
||||
|
||||
// Check if both frame around preference/trade-off vs hard constraint boundary
|
||||
const isPreferenceConstraintFrame = (q) =>
|
||||
/preference|trade.?off|constraint|boundary|absolute.*risk|no.*increase.*(risk|loss)/i.test(q);
|
||||
|
||||
// Check if both frame around priority ordering between growth and risk
|
||||
const isPriorityOrderingFrame = (q) =>
|
||||
/priority|which.*more.*important|priorit.*growth|growth.*versus.*risk|trade.*growth/i.test(q);
|
||||
|
||||
const aIsPC = isPreferenceConstraintFrame(a);
|
||||
const bIsPC = isPreferenceConstraintFrame(b);
|
||||
const aIsPriority = isPriorityOrderingFrame(a);
|
||||
const bIsPriority = isPriorityOrderingFrame(b);
|
||||
|
||||
// If one is preference/constraint and the other is priority ordering, they are materially different
|
||||
if (aIsPC !== bIsPC && !(aIsPC === bIsPriority)) {
|
||||
return "questions_materially_different";
|
||||
}
|
||||
|
||||
// Check: does one turn a preference/constraint distinction into simple priority?
|
||||
if ((aIsPC && bIsPriority) || (bIsPC && aIsPriority)) {
|
||||
return "questions_materially_different";
|
||||
}
|
||||
|
||||
// If both are same frame type, check if they address the same decision point
|
||||
if (aIsPC && bIsPC) {
|
||||
const bothAboutRiskBoundary = /risk|constraint|boundary/i.test(a) && /risk|constraint|boundary/i.test(b);
|
||||
if (bothAboutRiskBoundary) return "questions_materially_equivalent";
|
||||
}
|
||||
|
||||
// If neither matches the key frames, do manual-style overlap check
|
||||
// Check for shared core concepts: growth, risk, decision/choose/pick
|
||||
const sharedCore = [/growth/i.test(a) && /growth/i.test(b) && /risk/i.test(a) && /risk/i.test(b)];
|
||||
if (sharedCore) return "questions_materially_equivalent";
|
||||
|
||||
return "questions_materially_different";
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine whether two resolved meanings are materially equivalent.
|
||||
*/
|
||||
function classifyResolutionEquivalence(rA, rB) {
|
||||
const a = rA.toLowerCase();
|
||||
const b = rB.toLowerCase();
|
||||
|
||||
// Both agree risk avoidance is a hard boundary/constraint/absolute limit?
|
||||
const isHardConstraintMeaning = (r) =>
|
||||
/hard.*(constraint|boundary|limit)|no.*(increase|addition).*risk|absolut.*no.*risk|must.*(not|avoid)/i.test(r);
|
||||
|
||||
// Both say risk avoidance is a trade-off/preference?
|
||||
const isPreferenceMeaning = (r) =>
|
||||
/preference|trade.?off|willing.*trade|condition|depends/i.test(r);
|
||||
|
||||
const aIsHard = isHardConstraintMeaning(a);
|
||||
const bIsHard = isHardConstraintMeaning(b);
|
||||
const aIsPref = isPreferenceMeaning(a);
|
||||
const bIsPref = isPreferenceMeaning(b);
|
||||
|
||||
if (aIsHard === bIsHard && aIsPref === bIsPref) {
|
||||
return "resolutions_materially_equivalent";
|
||||
}
|
||||
|
||||
// If one resolved to hard constraint and the other to preference -- materially different
|
||||
if ((aIsHard && bIsPref) || (bIsHard && aIsPref)) {
|
||||
return "resolutions_materially_different";
|
||||
}
|
||||
|
||||
// Check broader semantic overlap on risk-avoidance meaning
|
||||
const sharedRiskWords = /risk/i.test(a) && /risk/i.test(b);
|
||||
const sharedConstraintWords = /(constraint|boundary|limit|absolute)/i.test(a) && /(constraint|boundary|limit|absolute)/i.test(b);
|
||||
|
||||
if (sharedConstraintWords && sharedRiskWords) return "resolutions_materially_equivalent";
|
||||
|
||||
// Manual-style: check if the core meaning (risk is not acceptable) is present in both
|
||||
const bothSayRiskNotAcceptable = /no.*(increase|accept|more).*risk/i.test(a) && /no.*(increase|accept|more).*risk/i.test(b);
|
||||
if (bothSayRiskNotAcceptable) return "resolutions_materially_equivalent";
|
||||
|
||||
// If one says risk is constraint and the other just mentions priority -- might still be equivalent
|
||||
// in terms of downstream consequence (user does not want more risk either way)
|
||||
const aHasCore = /no|hard|constraint|limit|absolute/i.test(a);
|
||||
const bHasCore = /no|hard|constraint|limit|absolute/i.test(b);
|
||||
if (aHasCore && bHasCore) return "resolutions_materially_equivalent";
|
||||
|
||||
return "resolutions_materially_different";
|
||||
}
|
||||
|
||||
function hasUnsupportedInference(resolvedMeaning, source) {
|
||||
const m = resolvedMeaning.toLowerCase();
|
||||
// Check for meanings that go beyond the fixed answer and source
|
||||
const unsupportedPatterns = [
|
||||
"growth is impossible",
|
||||
"cannot grow",
|
||||
"should not grow",
|
||||
"specific risk type",
|
||||
"which risk",
|
||||
"budget of",
|
||||
"financial loss",
|
||||
"revenue",
|
||||
"profit",
|
||||
"cost of",
|
||||
"which option",
|
||||
];
|
||||
const found = unsupportedPatterns.filter((p) => m.includes(p));
|
||||
return found.length > 0 ? found : null;
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Test suite
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("Experiment 54Y - Clarification Target Specificity Consequence", () => {
|
||||
const results = {
|
||||
variantA: { question: null, resolution: null },
|
||||
variantB: { question: null, resolution: null },
|
||||
timings: [],
|
||||
};
|
||||
|
||||
// -- Stage 1: Generate one question per variant --
|
||||
|
||||
it("Stage 1A: generate clarification question from precise target", async () => {
|
||||
const start = Date.now();
|
||||
const result = await callClarificationQuestion(SOURCE, VARIANT_A_TARGET);
|
||||
const elapsed = Date.now() - start;
|
||||
results.timings.push({ stage: "1A-question", ms: elapsed });
|
||||
|
||||
expect(result).toHaveProperty("question");
|
||||
expect(typeof result.question).toBe("string");
|
||||
expect(result.question.trim().endsWith("?")).toBe(true);
|
||||
|
||||
results.variantA.question = result.question;
|
||||
}, 120000);
|
||||
|
||||
it("Stage 1B: generate clarification question from broadened target", async () => {
|
||||
const start = Date.now();
|
||||
const result = await callClarificationQuestion(SOURCE, VARIANT_B_TARGET);
|
||||
const elapsed = Date.now() - start;
|
||||
results.timings.push({ stage: "1B-question", ms: elapsed });
|
||||
|
||||
expect(result).toHaveProperty("question");
|
||||
expect(typeof result.question).toBe("string");
|
||||
expect(result.question.trim().endsWith("?")).toBe(true);
|
||||
|
||||
results.variantB.question = result.question;
|
||||
}, 120000);
|
||||
|
||||
// -- Stage 2: Resolve same answer per variant --
|
||||
|
||||
it("Stage 2A: resolve fixed answer against Variant A question", async () => {
|
||||
const start = Date.now();
|
||||
const result = await callClarificationAnswerResolution(
|
||||
SOURCE,
|
||||
VARIANT_A_TARGET,
|
||||
results.variantA.question,
|
||||
FIXED_USER_ANSWER
|
||||
);
|
||||
const elapsed = Date.now() - start;
|
||||
results.timings.push({ stage: "2A-resolution", ms: elapsed });
|
||||
|
||||
expect(result).toHaveProperty("resolvedMeaning");
|
||||
expect(typeof result.resolvedMeaning).toBe("string");
|
||||
expect(typeof result.targetResolved).toBe("boolean");
|
||||
|
||||
results.variantA.resolution = result;
|
||||
}, 120000);
|
||||
|
||||
it("Stage 2B: resolve fixed answer against Variant B question", async () => {
|
||||
const start = Date.now();
|
||||
const result = await callClarificationAnswerResolution(
|
||||
SOURCE,
|
||||
VARIANT_B_TARGET,
|
||||
results.variantB.question,
|
||||
FIXED_USER_ANSWER
|
||||
);
|
||||
const elapsed = Date.now() - start;
|
||||
results.timings.push({ stage: "2B-resolution", ms: elapsed });
|
||||
|
||||
expect(result).toHaveProperty("resolvedMeaning");
|
||||
expect(typeof result.resolvedMeaning).toBe("string");
|
||||
expect(typeof result.targetResolved).toBe("boolean");
|
||||
|
||||
results.variantB.resolution = result;
|
||||
}, 120000);
|
||||
|
||||
// -- Aggregate evaluation --
|
||||
|
||||
it("54Y: evaluate question equivalence", () => {
|
||||
const eq = classifyQuestionEquivalence(results.variantA.question, results.variantB.question);
|
||||
results.questionEquivalence = eq;
|
||||
console.log("\n=== Experiment 54Y Results ===");
|
||||
console.log("\n--- Source ---");
|
||||
console.log(SOURCE);
|
||||
console.log("\n--- Fixed User Answer ---");
|
||||
console.log(FIXED_USER_ANSWER);
|
||||
console.log("\n--- Variant A (Precise Target) ---");
|
||||
console.log("Target:", VARIANT_A_TARGET);
|
||||
console.log("Generated question:", results.variantA.question);
|
||||
console.log("--- Variant B (Broadened Target) ---");
|
||||
console.log("Target:", VARIANT_B_TARGET);
|
||||
console.log("Generated question:", results.variantB.question);
|
||||
console.log("\n--- Question Equivalence: " + eq + " ---");
|
||||
});
|
||||
|
||||
it("54Y: evaluate resolution equivalence", () => {
|
||||
const eq = classifyResolutionEquivalence(results.variantA.resolution.resolvedMeaning, results.variantB.resolution.resolvedMeaning);
|
||||
results.resolutionEquivalence = eq;
|
||||
|
||||
console.log("\n--- Variant A Resolution ---");
|
||||
console.log("resolvedMeaning:", results.variantA.resolution.resolvedMeaning);
|
||||
console.log("targetResolved:", results.variantA.resolution.targetResolved);
|
||||
console.log("remainingUncertainty:", results.variantA.resolution.remainingUncertainty ?? null);
|
||||
|
||||
console.log("\n--- Variant B Resolution ---");
|
||||
console.log("resolvedMeaning:", results.variantB.resolution.resolvedMeaning);
|
||||
console.log("targetResolved:", results.variantB.resolution.targetResolved);
|
||||
console.log("remainingUncertainty:", results.variantB.resolution.remainingUncertainty ?? null);
|
||||
|
||||
console.log("\n--- Resolution Equivalence: " + eq + " ---");
|
||||
|
||||
// Check for unsupported inferences
|
||||
const aUnsupported = hasUnsupportedInference(results.variantA.resolution.resolvedMeaning, SOURCE);
|
||||
const bUnsupported = hasUnsupportedInference(results.variantB.resolution.resolvedMeaning, SOURCE);
|
||||
if (aUnsupported) console.log("Variant A unsupported inferences:", aUnsupported);
|
||||
if (bUnsupported) console.log("Variant B unsupported inferences:", bUnsupported);
|
||||
|
||||
// Timing summary
|
||||
const totalMs = results.timings.reduce((s, t) => s + t.ms, 0);
|
||||
const msArr = results.timings.map((t) => t.ms);
|
||||
console.log("\n--- Timing ---");
|
||||
console.log("Calls:", results.timings.length);
|
||||
console.log("Total:", totalMs + "ms");
|
||||
console.log("Average:", (totalMs / results.timings.length).toFixed(1) + "ms per call");
|
||||
console.log("Fastest:", Math.min(...msArr) + "ms");
|
||||
console.log("Slowest:", Math.max(...msArr) + "ms");
|
||||
|
||||
// Store for later review
|
||||
results.totalCalls = results.timings.length;
|
||||
results.totalTimeMs = totalMs;
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user