577 lines
27 KiB
JavaScript
577 lines
27 KiB
JavaScript
/**
|
|
* Experiment 52H — Does Ambiguity Fail Because of the Word "Important," or Because the Model Resists `cannot_determine` More Generally?
|
|
*
|
|
* Passive ambiguity-language experiment. Holds domain and subject constant (market-entry / customer-demand)
|
|
* and varies only the ambiguous wording pattern. Tests whether the model converts different neutral phrases
|
|
* into decisive relevance categories, or whether it preserves `cannot_determine`.
|
|
*
|
|
* Five fixed relationship statements with identical underlying subject and decision.
|
|
* Uses exactly the same category definitions and normalisation instruction as Experiment 52G.
|
|
* Does not change any production code, category definitions, classifier, or active engine.
|
|
*
|
|
* Hypothesis: If `important` is the main cause, weaker phrasings may preserve `cannot_determine`.
|
|
* If the model generally dislikes leaving relevance unresolved, all phrasings will still strengthen.
|
|
*/
|
|
|
|
import dotenv from "dotenv";
|
|
dotenv.config({ path: ".env.local" });
|
|
|
|
import { describe, it, expect, beforeAll } from "vitest";
|
|
|
|
/* ═══════════════════════════════════════════════════════════
|
|
* Enum categories (unchanged from production contract)
|
|
* ═══════════════════════════════════════════════════════════ */
|
|
|
|
const ENUM_CATEGORIES = [
|
|
"could_change_decision",
|
|
"supports_decision",
|
|
"unlikely_to_change_decision",
|
|
"cannot_determine",
|
|
];
|
|
|
|
/* ═══════════════════════════════════════════════════════════
|
|
* Category definitions — identical to Experiment 52G and production
|
|
* ═══════════════════════════════════════════════════════════ */
|
|
|
|
const CATEGORY_DEFINITIONS = {
|
|
could_change_decision:
|
|
"Answering could reasonably reverse the proposed action — it is a go/no-go condition or materially affects viability.",
|
|
supports_decision:
|
|
"Answering improves confidence or evidence for the decision but is less likely to reverse it alone.",
|
|
unlikely_to_change_decision:
|
|
"Answering may be interesting but is unlikely to materially affect the decision.",
|
|
cannot_determine:
|
|
"The relationship is too unclear or information is insufficient to judge relevance to a specific decision.",
|
|
};
|
|
|
|
/* ═══════════════════════════════════════════════════════════
|
|
* Normalisation instruction — identical to Experiment 52G (unchanged)
|
|
* ═══════════════════════════════════════════════════════════ */
|
|
|
|
const NORMALISATION_INSTRUCTION = `You are given a short statement describing how an unanswered question relates to a decision. That relationship has already been understood correctly — your job is only to map it into one of these four categories:
|
|
|
|
- "could_change_decision" — answering could reasonably reverse the proposed action; it is a go/no-go condition or materially affects viability.
|
|
- "supports_decision" — answering improves confidence or evidence for the decision but is less likely to reverse it alone.
|
|
- "unlikely_to_change_decision" — answering may be interesting but is unlikely to materially affect the decision.
|
|
- "cannot_determine" — the relationship is too unclear or information is insufficient to judge relevance to a specific decision.
|
|
|
|
Do not reinterpret the original situation — you have not been given it. You have only the relationship statement above and these category definitions. Choose the category that best matches the relationship statement.
|
|
|
|
Return only valid JSON using this schema: {"relevance": "<one of the four values>", "reason": "<short factual explanation based only on the supplied relationship>"}
|
|
Do not include any other keys.`;
|
|
|
|
/* ═══════════════════════════════════════════════════════════
|
|
* Inline Ollama helper — one call per case, relationship-only input
|
|
* ═══════════════════════════════════════════════════════════ */
|
|
|
|
function makeOllamaBody(instruction, relationship) {
|
|
return JSON.stringify({
|
|
model: process.env.OLLAMA_MODEL || "qwen-claude:latest",
|
|
messages: [
|
|
{ role: "system", content: instruction },
|
|
{
|
|
role: "user",
|
|
content: `Relationship: "${relationship}"`,
|
|
},
|
|
],
|
|
format: "json",
|
|
stream: false,
|
|
});
|
|
}
|
|
|
|
async function callAmbiguousWordingTest(relationship) {
|
|
const baseUrl = process.env.OLLAMA_BASE_URL;
|
|
if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");
|
|
|
|
const model = process.env.OLLAMA_MODEL || "qwen-claude:latest";
|
|
const body = makeOllamaBody(NORMALISATION_INSTRUCTION, relationship);
|
|
const res = await fetch(`${baseUrl}/api/chat`, {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body,
|
|
signal: AbortSignal.timeout(120000),
|
|
});
|
|
if (!res.ok) throw new Error(`Ollama returned ${res.status}`);
|
|
const data = await res.json();
|
|
const rawText =
|
|
typeof data.message?.content === "string"
|
|
? data.message.content
|
|
: JSON.stringify(data.message?.content || {});
|
|
return { result: JSON.parse(rawText), model };
|
|
}
|
|
|
|
/* ═══════════════════════════════════════════════════════════
|
|
* Five fixed wording variants — same domain and subject, different wording
|
|
* All five communicate only "there is some relationship."
|
|
* None states how strong that relationship is.
|
|
* ═══════════════════════════════════════════════════════════ */
|
|
|
|
const CASES = [
|
|
{
|
|
id: "case1-important",
|
|
wording: "\"important to\"",
|
|
description: "Important to (control — same pattern as Exp 52G)",
|
|
relationship: "Understanding customer demand would be important to the market-entry decision.",
|
|
expectedEnum: "cannot_determine",
|
|
},
|
|
{
|
|
id: "case2-relevant",
|
|
wording: "\"relevant to\"",
|
|
description: "Relevant to",
|
|
relationship: "Understanding customer demand would be relevant to the market-entry decision.",
|
|
expectedEnum: "cannot_determine",
|
|
},
|
|
{
|
|
id: "case3-worth",
|
|
wording: "\"worth considering\"",
|
|
description: "Worth considering",
|
|
relationship: "Customer demand would be worth considering when making the market-entry decision.",
|
|
expectedEnum: "cannot_determine",
|
|
},
|
|
{
|
|
id: "case4-maymatter",
|
|
wording: "\"may matter for\"",
|
|
description: "May matter for",
|
|
relationship: "Customer demand may matter for the market-entry decision.",
|
|
expectedEnum: "cannot_determine",
|
|
},
|
|
{
|
|
id: "case5-connected",
|
|
wording: "\"connected to\"",
|
|
description: "Connected to",
|
|
relationship: "Customer demand is connected to the market-entry decision.",
|
|
expectedEnum: "cannot_determine",
|
|
},
|
|
];
|
|
|
|
/* ═══════════════════════════════════════════════════════════
|
|
* Grounding diagnostic labels — test-only, not production
|
|
* ═══════════════════════════════════════════════════════════ */
|
|
|
|
const GROUNDING_LABELS = {
|
|
GROUNDED_ONLY: "grounded_only_in_statement",
|
|
STRENGTHENED: "introduced_stronger_relationship",
|
|
};
|
|
|
|
function checkGrounding(expectedEnum, reason) {
|
|
if (expectedEnum !== "cannot_determine") return GROUNDING_LABELS.GROUNDED_ONLY;
|
|
|
|
const reasonLower = (reason || "").toLowerCase();
|
|
const strongerSignals = [
|
|
/material.*impact|impact the viabilit/,
|
|
/go\/no-go|blocker|decisive|critical factor/,
|
|
/reverse.*decision|reverses?.*action|change.*outcome/i,
|
|
/confidence.*strengthen|increase.*confident|matters for.*viable/,
|
|
/must.*satisfy|mandatory|essential.*condition|necessary.*for.*entry/,
|
|
/determines?.*viability|determine.*whether.*enter|determines?.*the decision/,
|
|
/unlikel?.*to.*proceed|prevent.*entry|block.*market/i,
|
|
];
|
|
|
|
const hasStronger = strongerSignals.some((p) => p.test(reasonLower));
|
|
return hasStronger ? GROUNDING_LABELS.STRENGTHENED : GROUNDING_LABELS.GROUNDED_ONLY;
|
|
}
|
|
|
|
/* ═══════════════════════════════════════════════════════════
|
|
* Results holder — populated by beforeAll (5 calls total)
|
|
* ═══════════════════════════════════════════════════════════ */
|
|
|
|
let experimentResults = {};
|
|
let inferenceCount = 0;
|
|
let timingStats = { min: Infinity, max: 0, total: 0 };
|
|
let modelFailureReason = null;
|
|
|
|
beforeAll(async () => {
|
|
experimentResults = {};
|
|
|
|
for (const c of CASES) {
|
|
let result = null;
|
|
let latency = 0;
|
|
const t0 = Date.now();
|
|
try {
|
|
result = await callAmbiguousWordingTest(c.relationship);
|
|
latency = Date.now() - t0;
|
|
} catch (e) {
|
|
modelFailureReason = `case ${c.id}: ${e.message}`;
|
|
result = { result: null };
|
|
}
|
|
timingStats.min = Math.min(timingStats.min, latency);
|
|
timingStats.max = Math.max(timingStats.max, latency);
|
|
timingStats.total += latency;
|
|
|
|
experimentResults[c.id] = {
|
|
wording: c.wording,
|
|
description: c.description,
|
|
relationship: c.relationship,
|
|
expectedEnum: c.expectedEnum,
|
|
returnedEnum: result.result?.relevance || "error",
|
|
reason: result.result?.reason || "none",
|
|
model: result.model,
|
|
latencyMs: latency,
|
|
};
|
|
inferenceCount += 1;
|
|
}
|
|
}, 600000);
|
|
|
|
/* ═══════════════════════════════════════════════════════════
|
|
* Infrastructure assertions — exactly 5 calls, same config, production unchanged
|
|
* ═══════════════════════════════════════════════════════════ */
|
|
|
|
describe("Experiment 52H — Infrastructure", () => {
|
|
it("uses Ollama config from .env.local", () => {
|
|
expect(process.env.OLLAMA_BASE_URL).toBeTruthy();
|
|
expect(process.env.OLLAMA_MODEL).toBe("qwen-claude:latest");
|
|
});
|
|
|
|
it("all returned enums belong to the existing four-category contract", () => {
|
|
for (const c of CASES) {
|
|
const r = experimentResults[c.id]?.returnedEnum;
|
|
expect(ENUM_CATEGORIES).toContain(r);
|
|
}
|
|
});
|
|
|
|
it("all cases include a reason string", () => {
|
|
for (const c of CASES) {
|
|
const r = experimentResults[c.id]?.reason;
|
|
expect(typeof r).toBe("string");
|
|
expect(r.length).toBeGreaterThan(0);
|
|
}
|
|
});
|
|
|
|
it("same Ollama host used throughout", () => {
|
|
expect(process.env.OLLAMA_BASE_URL).toBe("http://192.168.1.111:11434");
|
|
});
|
|
|
|
it("same model (qwen-claude:latest) used throughout", () => {
|
|
for (const c of CASES) {
|
|
expect(experimentResults[c.id]?.model).toBe("qwen-claude:latest");
|
|
}
|
|
});
|
|
|
|
it("exactly 5 live inference calls were made", () => {
|
|
expect(inferenceCount).toBe(5);
|
|
});
|
|
|
|
it("normalisation instruction is identical for all five calls", () => {
|
|
expect(typeof NORMALISATION_INSTRUCTION).toBe("string");
|
|
expect(NORMALISATION_INSTRUCTION.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("fixed expected enums before any live call — all five are cannot_determine", () => {
|
|
let ambiguousCount = 0;
|
|
for (const c of CASES) {
|
|
if (c.expectedEnum === "cannot_determine") ambiguousCount++;
|
|
}
|
|
expect(ambiguousCount).toBe(5);
|
|
});
|
|
|
|
it("all five cases use the same domain and subject — only wording changes", () => {
|
|
// Verify all cases refer to customer demand / market-entry
|
|
for (const c of CASES) {
|
|
expect(c.relationship).toContain("market-entry");
|
|
}
|
|
// Verify wording differs between cases
|
|
const wordings = CASES.map((c) => c.wording);
|
|
const uniqueWordings = [...new Set(wordings)];
|
|
expect(uniqueWordings.length).toBe(5);
|
|
});
|
|
|
|
it("only the relationship statement is supplied to each case", () => {
|
|
for (const c of CASES) {
|
|
expect(c.relationship).toBeTruthy();
|
|
expect(typeof c.relationship).toBe("string");
|
|
expect(c).not.toHaveProperty("decisionTarget");
|
|
expect(c).not.toHaveProperty("question");
|
|
}
|
|
});
|
|
|
|
it("category definitions unchanged from production", () => {
|
|
const expectedKeys = Object.keys(CATEGORY_DEFINITIONS);
|
|
expect(expectedKeys).toContain("could_change_decision");
|
|
expect(expectedKeys).toContain("supports_decision");
|
|
expect(expectedKeys).toContain("unlikely_to_change_decision");
|
|
expect(expectedKeys).toContain("cannot_determine");
|
|
|
|
expect(CATEGORY_DEFINITIONS.could_change_decision).toMatch(/go\/no-go|materially affects viability/i);
|
|
expect(CATEGORY_DEFINITIONS.supports_decision).toMatch(/improves confidence|less likely to reverse/i);
|
|
});
|
|
|
|
it("instruction and definitions unchanged from Experiment 52G", () => {
|
|
const expectedInstruction = `You are given a short statement describing how an unanswered question relates to a decision. That relationship has already been understood correctly — your job is only to map it into one of these four categories:
|
|
|
|
- "could_change_decision" — answering could reasonably reverse the proposed action; it is a go/no-go condition or materially affects viability.
|
|
- "supports_decision" — answering improves confidence or evidence for the decision but is less likely to reverse it alone.
|
|
- "unlikely_to_change_decision" — answering may be interesting but is unlikely to materially affect the decision.
|
|
- "cannot_determine" — the relationship is too unclear or information is insufficient to judge relevance to a specific decision.
|
|
|
|
Do not reinterpret the original situation — you have not been given it. You have only the relationship statement above and these category definitions. Choose the category that best matches the relationship statement.
|
|
|
|
Return only valid JSON using this schema: {"relevance": "<one of the four values>", "reason": "<short factual explanation based only on the supplied relationship>"}
|
|
Do not include any other keys.`;
|
|
expect(NORMALISATION_INSTRUCTION).toBe(expectedInstruction);
|
|
});
|
|
});
|
|
|
|
/* ═══════════════════════════════════════════════════════════
|
|
* Case 1 — "important to" (control)
|
|
* ═══════════════════════════════════════════════════════════ */
|
|
|
|
describe("Experiment 52H — Case 1: \"important to\"", () => {
|
|
it("returns its enum result within the contract", () => {
|
|
const r = experimentResults["case1-important"];
|
|
expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
|
|
expect(typeof r.reason).toBe("string");
|
|
expect(r.reason.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("expected enum is cannot_determine", () => {
|
|
const r = experimentResults["case1-important"];
|
|
expect(r.expectedEnum).toBe("cannot_determine");
|
|
});
|
|
|
|
it("match/mismatch check", () => {
|
|
const r = experimentResults["case1-important"];
|
|
const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
|
|
console.log(`[important to] expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match}`);
|
|
});
|
|
|
|
it("grounding check", () => {
|
|
const r = experimentResults["case1-important"];
|
|
const grounding = checkGrounding(r.expectedEnum, r.reason);
|
|
console.log(`[important to] grounding: ${grounding} | reason="${r.reason}"`);
|
|
expect(grounding).toBeTruthy();
|
|
});
|
|
});
|
|
|
|
/* ═══════════════════════════════════════════════════════════
|
|
* Case 2 — "relevant to"
|
|
* ═══════════════════════════════════════════════════════════ */
|
|
|
|
describe("Experiment 52H — Case 2: \"relevant to\"", () => {
|
|
it("returns its enum result within the contract", () => {
|
|
const r = experimentResults["case2-relevant"];
|
|
expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
|
|
expect(typeof r.reason).toBe("string");
|
|
expect(r.reason.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("expected enum is cannot_determine", () => {
|
|
const r = experimentResults["case2-relevant"];
|
|
expect(r.expectedEnum).toBe("cannot_determine");
|
|
});
|
|
|
|
it("match/mismatch check", () => {
|
|
const r = experimentResults["case2-relevant"];
|
|
const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
|
|
console.log(`[relevant to] expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match}`);
|
|
});
|
|
|
|
it("grounding check", () => {
|
|
const r = experimentResults["case2-relevant"];
|
|
const grounding = checkGrounding(r.expectedEnum, r.reason);
|
|
console.log(`[relevant to] grounding: ${grounding} | reason="${r.reason}"`);
|
|
expect(grounding).toBeTruthy();
|
|
});
|
|
});
|
|
|
|
/* ═══════════════════════════════════════════════════════════
|
|
* Case 3 — "worth considering"
|
|
* ═══════════════════════════════════════════════════════════ */
|
|
|
|
describe("Experiment 52H — Case 3: \"worth considering\"", () => {
|
|
it("returns its enum result within the contract", () => {
|
|
const r = experimentResults["case3-worth"];
|
|
expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
|
|
expect(typeof r.reason).toBe("string");
|
|
expect(r.reason.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("expected enum is cannot_determine", () => {
|
|
const r = experimentResults["case3-worth"];
|
|
expect(r.expectedEnum).toBe("cannot_determine");
|
|
});
|
|
|
|
it("match/mismatch check", () => {
|
|
const r = experimentResults["case3-worth"];
|
|
const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
|
|
console.log(`[worth considering] expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match}`);
|
|
});
|
|
|
|
it("grounding check", () => {
|
|
const r = experimentResults["case3-worth"];
|
|
const grounding = checkGrounding(r.expectedEnum, r.reason);
|
|
console.log(`[worth considering] grounding: ${grounding} | reason="${r.reason}"`);
|
|
expect(grounding).toBeTruthy();
|
|
});
|
|
});
|
|
|
|
/* ═══════════════════════════════════════════════════════════
|
|
* Case 4 — "may matter for"
|
|
* ═══════════════════════════════════════════════════════════ */
|
|
|
|
describe("Experiment 52H — Case 4: \"may matter for\"", () => {
|
|
it("returns its enum result within the contract", () => {
|
|
const r = experimentResults["case4-maymatter"];
|
|
expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
|
|
expect(typeof r.reason).toBe("string");
|
|
expect(r.reason.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("expected enum is cannot_determine", () => {
|
|
const r = experimentResults["case4-maymatter"];
|
|
expect(r.expectedEnum).toBe("cannot_determine");
|
|
});
|
|
|
|
it("match/mismatch check", () => {
|
|
const r = experimentResults["case4-maymatter"];
|
|
const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
|
|
console.log(`[may matter for] expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match}`);
|
|
});
|
|
|
|
it("grounding check", () => {
|
|
const r = experimentResults["case4-maymatter"];
|
|
const grounding = checkGrounding(r.expectedEnum, r.reason);
|
|
console.log(`[may matter for] grounding: ${grounding} | reason="${r.reason}"`);
|
|
expect(grounding).toBeTruthy();
|
|
});
|
|
});
|
|
|
|
/* ═══════════════════════════════════════════════════════════
|
|
* Case 5 — "connected to"
|
|
* ═══════════════════════════════════════════════════════════ */
|
|
|
|
describe("Experiment 52H — Case 5: \"connected to\"", () => {
|
|
it("returns its enum result within the contract", () => {
|
|
const r = experimentResults["case5-connected"];
|
|
expect(ENUM_CATEGORIES).toContain(r.returnedEnum);
|
|
expect(typeof r.reason).toBe("string");
|
|
expect(r.reason.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("expected enum is cannot_determine", () => {
|
|
const r = experimentResults["case5-connected"];
|
|
expect(r.expectedEnum).toBe("cannot_determine");
|
|
});
|
|
|
|
it("match/mismatch check", () => {
|
|
const r = experimentResults["case5-connected"];
|
|
const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
|
|
console.log(`[connected to] expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match}`);
|
|
});
|
|
|
|
it("grounding check", () => {
|
|
const r = experimentResults["case5-connected"];
|
|
const grounding = checkGrounding(r.expectedEnum, r.reason);
|
|
console.log(`[connected to] grounding: ${grounding} | reason="${r.reason}"`);
|
|
expect(grounding).toBeTruthy();
|
|
});
|
|
});
|
|
|
|
/* ═══════════════════════════════════════════════════════════
|
|
* Cross-wording comparison — does wording change classification?
|
|
* ═══════════════════════════════════════════════════════════ */
|
|
|
|
describe("Experiment 52H — Cross-wording analysis", () => {
|
|
it("reports all matches and mismatches", () => {
|
|
for (const c of CASES) {
|
|
const r = experimentResults[c.id];
|
|
const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
|
|
console.log(
|
|
`[${c.wording}] ${c.description}: expected=${r.expectedEnum} | returned=${r.returnedEnum} | ${match} | ` +
|
|
`reason="${r.reason}" | latency=${r.latencyMs}ms`
|
|
);
|
|
}
|
|
});
|
|
|
|
it("cannot_determine count for all five cases", () => {
|
|
let canNotDetermineCount = 0;
|
|
for (const c of CASES) {
|
|
const r = experimentResults[c.id];
|
|
if (r.returnedEnum === "cannot_determine") canNotDetermineCount++;
|
|
}
|
|
console.log(`Cannot_determine count: ${canNotDetermineCount}/5`);
|
|
expect(typeof canNotDetermineCount).toBe("number");
|
|
});
|
|
|
|
it("did \"important to\" (control) become could_change_decision?", () => {
|
|
const r = experimentResults["case1-important"];
|
|
const isCouldChange = r.returnedEnum === "could_change_decision";
|
|
console.log(`[important to] returned could_change_decision: ${isCouldChange}`);
|
|
expect(typeof isCouldChange).toBe("boolean");
|
|
});
|
|
|
|
it("did each weaker phrasing preserve cannot_determine?", () => {
|
|
const preserved = ["case2-relevant", "case3-worth", "case4-maymatter", "case5-connected"].map(
|
|
(id) => experimentResults[id].returnedEnum === "cannot_determine"
|
|
);
|
|
const allPreserved = preserved.every((v) => v);
|
|
console.log(`All weaker phrasings preserved cannot_determine: ${allPreserved}`);
|
|
console.log(` relevant_to=${experimentResults["case2-relevant"].returnedEnum}`);
|
|
console.log(` worth_considering=${experimentResults["case3-worth"].returnedEnum}`);
|
|
console.log(` may_matter_for=${experimentResults["case4-maymatter"].returnedEnum}`);
|
|
console.log(` connected_to=${experimentResults["case5-connected"].returnedEnum}`);
|
|
expect(typeof allPreserved).toBe("boolean");
|
|
});
|
|
|
|
it("did different wording produce different enum categories?", () => {
|
|
const categories = CASES.map((c) => experimentResults[c.id].returnedEnum);
|
|
const uniqueCategories = [...new Set(categories)];
|
|
const hadDivergence = uniqueCategories.length > 1;
|
|
console.log(`Unique categories across wordings: ${uniqueCategories.join(", ")} | divergence: ${hadDivergence}`);
|
|
expect(typeof hadDivergence).toBe("boolean");
|
|
});
|
|
|
|
it("count of cases where model introduced stronger relationship", () => {
|
|
let strengthenedCount = 0;
|
|
for (const c of CASES) {
|
|
const r = experimentResults[c.id];
|
|
if (r.expectedEnum === "cannot_determine") {
|
|
const grounding = checkGrounding(r.expectedEnum, r.reason);
|
|
if (grounding === GROUNDING_LABELS.STRENGTHENED) strengthenedCount++;
|
|
}
|
|
}
|
|
console.log(`Cases with stronger relationship introduced: ${strengthenedCount}/5`);
|
|
expect(typeof strengthenedCount).toBe("number");
|
|
});
|
|
|
|
it("full output log", () => {
|
|
console.log("\n=== Experiment 52H Summary ===");
|
|
for (const c of CASES) {
|
|
const r = experimentResults[c.id];
|
|
const match = r.returnedEnum === r.expectedEnum ? "match" : "mismatch";
|
|
const grounding = checkGrounding(r.expectedEnum, r.reason);
|
|
console.log(
|
|
`[${c.wording}] ${r.expectedEnum} → ${r.returnedEnum} (${match}) | ` +
|
|
`reason="${r.reason}" | grounding=${grounding} | latency=${r.latencyMs}ms`
|
|
);
|
|
}
|
|
});
|
|
});
|
|
|
|
/* ═══════════════════════════════════════════════════════════
|
|
* Inference timing (observational only)
|
|
* ═══════════════════════════════════════════════════════════ */
|
|
|
|
describe("Experiment 52H — Inference timing", () => {
|
|
it("records min, max, total timing for all 5 calls", () => {
|
|
expect(timingStats.min).toBeGreaterThan(0);
|
|
expect(timingStats.max).toBeGreaterThanOrEqual(timingStats.min);
|
|
expect(timingStats.total).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("records average latency within reasonable range", () => {
|
|
const avg = timingStats.total / 5;
|
|
expect(avg).toBeGreaterThan(5000);
|
|
expect(avg).toBeLessThan(120000);
|
|
});
|
|
|
|
it("logs timing summary", () => {
|
|
const avg = Math.round(timingStats.total / 5);
|
|
console.log(`\n=== Experiment 52H Timing ===`);
|
|
console.log(`Calls: 5`);
|
|
console.log(`Total: ${timingStats.total}ms`);
|
|
console.log(`Average: ${avg}ms`);
|
|
console.log(`Fastest: ${timingStats.min}ms`);
|
|
console.log(`Slowest: ${timingStats.max}ms`);
|
|
});
|
|
});
|