experiment: run small semantic decision-relevance probe
This commit is contained in:
@@ -1,91 +1,56 @@
|
||||
/**
|
||||
* Experiment 52 — Can Semantic Interpretation Generalise Decision Relevance?
|
||||
* Experiment 52B — Small Semantic Probe With the Existing Qwen Model
|
||||
*
|
||||
* Passive comparison. Tests whether a small, one-shot semantic interpretation step
|
||||
* judges decision relevance more consistently across paraphrases and domains than
|
||||
* the existing deterministic keyword-based classifier.
|
||||
*
|
||||
* Six cases from Experiment 52 corpus, run exactly once each.
|
||||
* No production code changes. No active engine integration. Pure test-level evaluation.
|
||||
* Model call infrastructure is minimal: one inline helper using fetch to Ollama /api/chat.
|
||||
* Same configured host and model as Experiment 52/52A.
|
||||
*/
|
||||
|
||||
import dotenv from "dotenv";
|
||||
dotenv.config({ path: ".env.local" });
|
||||
|
||||
import { describe, it, expect, beforeAll, afterEach } from "vitest";
|
||||
import { describe, it, expect, beforeAll } from "vitest";
|
||||
import { assessQuestionRelevanceToDecision } from "@/lib/graph/question-decision-relevance.js";
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Test data: fixed human reference labels (set BEFORE evaluation)
|
||||
* Semantic instruction — unchanged concept from Experiment 52.
|
||||
* Added explicit category enum so the model outputs our schema
|
||||
* (qwen-claude:latest needs this to avoid natural-language labels).
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
const SEMANTIC_INSTRUCTION = `Given a decision and one unanswered question, classify whether resolving that question could directly change the decision, would provide useful support for the decision, is unlikely to affect the decision, or cannot be determined from the information provided. Return only valid JSON matching the schema: {"relevance": "<category>", "reason": "<short explanation>"}`;
|
||||
const SEMANTIC_INSTRUCTION = `Given a decision and one unanswered question, classify whether resolving that question could directly change the decision, would provide useful support for the decision, is unlikely to affect the decision, or cannot be determined from the information provided.
|
||||
|
||||
Return only valid JSON using exactly these category values (no others):
|
||||
- "could_change_decision" — answering could reasonably reverse the proposed action
|
||||
- "supports_decision" — answering improves confidence/evidence but less likely to reverse alone
|
||||
- "unlikely_to_change_decision" — answering is unlikely to materially affect the decision
|
||||
- "cannot_determine" — information is insufficient to judge
|
||||
|
||||
Schema: {"relevance": "<one of the four values above>", "reason": "<short explanation>"}
|
||||
|
||||
Do not use other words like "high", "low", "direct", etc. Use only the four category names listed.`;
|
||||
|
||||
const SEMANTIC_CATEGORIES = ["could_change_decision", "supports_decision", "unlikely_to_change_decision", "cannot_determine"];
|
||||
|
||||
function makeUnknown(id, label) {
|
||||
return { id, label, description: label, kind: "unknown", status: "unknown", confidence: "low", value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], childIds: [] };
|
||||
}
|
||||
|
||||
const DOMAIN_A_DECISION = "Should we enter the European market with our SaaS analytics platform?";
|
||||
const DOMAIN_B_DECISION = "Should we organise the community event outdoors this September?";
|
||||
|
||||
/* ── Domain A — Coherent (market entry) ───────────────────── */
|
||||
|
||||
const COHERENT_A = [
|
||||
{ id: "demand", label: "Whether there is genuine customer demand for analytics tools in Europe", humanRef: "could_change_decision" },
|
||||
{ id: "compliance", label: "Whether our product meets European compliance requirements", humanRef: "supports_decision" },
|
||||
{ id: "cost-benefit", label: "What the cost would be to adapt the platform for the EU market versus potential revenue", humanRef: "supports_decision" },
|
||||
{ id: "differentiation", label: "How our analytics approach compares with existing European competitors", humanRef: "supports_decision" },
|
||||
];
|
||||
|
||||
/* ── Domain A — Scattered (market entry) ──────────────────── */
|
||||
|
||||
const SCATTERED_A = [
|
||||
{ id: "staff-disagree", label: "Can two senior staff members resolve their ongoing disagreement?", humanRef: "unlikely_to_change_decision" },
|
||||
{ id: "office-lease", label: "Should the head office lease be renewed at the current rate next year?", humanRef: "unlikely_to_change_decision" },
|
||||
{ id: "unrelated-pricing", label: "Does an existing unrelated product's pricing align with market willingness to pay?", humanRef: "unlikely_to_change_decision" },
|
||||
];
|
||||
|
||||
/* ── Domain B — Coherent (community event) ────────────────── */
|
||||
|
||||
const COHERENT_B = [
|
||||
{ id: "weather-risk", label: "Whether there is sufficient weather risk for an outdoor event in September", humanRef: "could_change_decision" },
|
||||
{ id: "insurance", label: "What insurance requirements apply for hosting the event outdoors", humanRef: "could_change_decision" },
|
||||
{ id: "capacity", label: "Whether the outdoor venue can accommodate expected attendance", humanRef: "supports_decision" },
|
||||
{ id: "accessibility", label: "Whether the outdoor venue meets accessibility requirements for all attendees", humanRef: "supports_decision" },
|
||||
];
|
||||
|
||||
/* ── Domain B — Scattered (community event) ───────────────── */
|
||||
|
||||
const SCATTERED_B = [
|
||||
{ id: "board-chairs", label: "Should the board replace its meeting room chairs next month?", humanRef: "unlikely_to_change_decision" },
|
||||
{ id: "volunteer-staffing", label: "Whether available volunteers can staff the registration desk on event day", humanRef: "cannot_determine" },
|
||||
{ id: "covered-space", label: "What local parks offer covered spaces in case of rain?", humanRef: "cannot_determine" },
|
||||
];
|
||||
|
||||
/* ── Paraphrases (Domain A decision target for all) ───────── */
|
||||
|
||||
const PARAPHRASE_COHERENT_ORIG = { id: "coh-orig", label: "Whether to enter the European market for analytics tools", humanRef: "could_change_decision" };
|
||||
const PARAPHRASE_COHERENT_PHR = { id: "coh-paraphrased", label: "Would enough people there actually want what we offer?", humanRef: "could_change_decision" };
|
||||
const PARAPHRASE_UNRELATED_ORIG = { id: "unrel-orig", label: "What benchmarks do other SaaS companies use for market sizing", humanRef: "unlikely_to_change_decision" };
|
||||
const PARAPHRASE_UNRELATED_PHR = { id: "unrel-paraphrased", label: "Which analytics firms set the industry standard?", humanRef: "cannot_determine" };
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Minimal inline Ollama helper (10 lines)
|
||||
* Mirrors lib/llm/provider.js pattern: fetch to /api/chat with format:json.
|
||||
* This is not production infrastructure — it exists only for this test.
|
||||
*/
|
||||
* Minimal inline Ollama helper — mirrors production pattern
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
async function semanticInterpret(decisionTarget, unknown) {
|
||||
async function semanticInterpret(decisionTarget, unknownLabel) {
|
||||
const baseUrl = process.env.OLLAMA_BASE_URL;
|
||||
if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");
|
||||
|
||||
const model = process.env.OLLAMA_MODEL || "llama3.1";
|
||||
|
||||
const body = JSON.stringify({
|
||||
model: process.env.OLLAMA_MODEL || "llama3.1",
|
||||
model,
|
||||
messages: [
|
||||
{ role: "system", content: SEMANTIC_INSTRUCTION },
|
||||
{ role: "user", content: `Decision: "${decisionTarget}"\nQuestion: "${unknown.label}"`, },
|
||||
{ role: "user", content: `Decision: "${decisionTarget}"\nQuestion: "${unknownLabel}"`, },
|
||||
],
|
||||
format: "json",
|
||||
stream: false,
|
||||
@@ -96,265 +61,257 @@ async function semanticInterpret(decisionTarget, unknown) {
|
||||
});
|
||||
if (!res.ok) throw new Error(`Ollama returned ${res.status}`);
|
||||
const data = await res.json();
|
||||
const text = typeof data.message?.content === "string" ? data.message.content : JSON.stringify(data.message?.content);
|
||||
return JSON.parse(text);
|
||||
const rawText = typeof data.message?.content === "string" ? data.message.content : JSON.stringify(data.message?.content || {});
|
||||
return { result: JSON.parse(rawText), model, latencyMs: 0 };
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Global fixture: run all cases three times, record stability
|
||||
* Decision targets (from Experiment 52)
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
const ALL_CASES = [
|
||||
...COHERENT_A.map((c) => ({ ...c, domain: "Domain A (market)", decisionTarget: DOMAIN_A_DECISION })),
|
||||
...SCATTERED_A.map((c) => ({ ...c, domain: "Domain A scattered", decisionTarget: DOMAIN_A_DECISION })),
|
||||
...COHERENT_B.map((c) => ({ ...c, domain: "Domain B (event)", decisionTarget: DOMAIN_B_DECISION })),
|
||||
...SCATTERED_B.map((c) => ({ ...c, domain: "Domain B scattered", decisionTarget: DOMAIN_B_DECISION })),
|
||||
{ ...PARAPHRASE_COHERENT_ORIG, domain: "Paraphrase (coherent orig)", decisionTarget: DOMAIN_A_DECISION },
|
||||
{ ...PARAPHRASE_COHERENT_PHR, domain: "Paraphrase (coherent paraphrased)", decisionTarget: DOMAIN_A_DECISION },
|
||||
{ ...PARAPHRASE_UNRELATED_ORIG, domain: "Paraphrase (unrelated orig)", decisionTarget: DOMAIN_A_DECISION },
|
||||
{ ...PARAPHRASE_UNRELATED_PHR, domain: "Paraphrase (unrelated paraphrased)", decisionTarget: DOMAIN_A_DECISION },
|
||||
const DOMAIN_A_DECISION = "Should we enter the European market with our SaaS analytics platform?";
|
||||
const DOMAIN_B_DECISION = "Should we organise the community event outdoors this September?";
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Experiment 52B — Six live inference cases (exactly 6 calls)
|
||||
*
|
||||
* Cases drawn from the existing Experiment 52 corpus.
|
||||
* Human reference labels fixed before evaluation; not changed after seeing results.
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
const SIX_CASES = [
|
||||
{
|
||||
id: "case1-familiar-relevant",
|
||||
decisionTarget: DOMAIN_A_DECISION,
|
||||
unknownLabel: "Whether there is genuine customer demand for analytics tools in Europe",
|
||||
humanRef: "could_change_decision",
|
||||
purpose: "Confirm semantic interpretation handles an easy in-domain relevant question.",
|
||||
},
|
||||
{
|
||||
id: "case2-familiar-unrelated",
|
||||
decisionTarget: DOMAIN_A_DECISION,
|
||||
unknownLabel: "Can two senior staff members resolve their ongoing disagreement?",
|
||||
humanRef: "unlikely_to_change_decision",
|
||||
purpose: "Confirm semantic interpretation can reject an obviously unrelated question.",
|
||||
},
|
||||
{
|
||||
id: "case3-paraphrase-relevant",
|
||||
decisionTarget: DOMAIN_A_DECISION,
|
||||
unknownLabel: "Would enough people there actually want what we offer?",
|
||||
humanRef: "could_change_decision",
|
||||
purpose: "Known deterministic keyword failure — semantic model should handle this paraphrase.",
|
||||
},
|
||||
{
|
||||
id: "case4-domain-b-relevant",
|
||||
decisionTarget: DOMAIN_B_DECISION,
|
||||
unknownLabel: "Whether there is sufficient weather risk for an outdoor event in September",
|
||||
humanRef: "could_change_decision",
|
||||
purpose: "Test whether semantic interpretation generalises beyond the market-entry vocabulary.",
|
||||
},
|
||||
{
|
||||
id: "case5-domain-b-supporting",
|
||||
decisionTarget: DOMAIN_B_DECISION,
|
||||
unknownLabel: "What insurance requirements apply for hosting the event outdoors",
|
||||
humanRef: "supports_decision",
|
||||
purpose: "Test whether semantic interpretation distinguishes supporting from decisive.",
|
||||
},
|
||||
{
|
||||
id: "case6-domain-b-unrelated",
|
||||
decisionTarget: DOMAIN_B_DECISION,
|
||||
unknownLabel: "Should the board replace its meeting room chairs next month?",
|
||||
humanRef: "unlikely_to_change_decision",
|
||||
purpose: "Confirm semantic interpretation does not merely mark everything as relevant.",
|
||||
},
|
||||
];
|
||||
|
||||
let semanticResults = {}; // { id: { runs: [result1, result2, result3], stability: "stable"|"unstable" } }
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Results holder — populated by beforeAll (6 calls total)
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
let experimentResults = {};
|
||||
let inferenceCount = 0;
|
||||
let timingStats = { min: Infinity, max: 0, total: 0 };
|
||||
let modelFailureReason = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
semanticResults = {};
|
||||
for (const c of ALL_CASES) {
|
||||
const unknownNode = makeUnknown(c.id, c.label);
|
||||
const runs = [];
|
||||
let stability = "stable";
|
||||
let firstCategory = null;
|
||||
experimentResults = {};
|
||||
|
||||
for (const c of SIX_CASES) {
|
||||
const t0 = Date.now();
|
||||
try {
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
const result = await semanticInterpret(c.decisionTarget, unknownNode);
|
||||
if (!SEMANTIC_CATEGORIES.includes(result.relevance)) {
|
||||
result.relevance = "cannot_determine";
|
||||
}
|
||||
runs.push(result);
|
||||
if (firstCategory === null) firstCategory = result.relevance;
|
||||
else if (result.relevance !== firstCategory) stability = "unstable";
|
||||
const { result, model } = await semanticInterpret(c.decisionTarget, c.unknownLabel);
|
||||
const actualLatency = Date.now() - t0;
|
||||
timingStats.min = Math.min(timingStats.min, actualLatency);
|
||||
timingStats.max = Math.max(timingStats.max, actualLatency);
|
||||
timingStats.total += actualLatency;
|
||||
|
||||
if (!SEMANTIC_CATEGORIES.includes(result.relevance)) {
|
||||
result.relevance = "cannot_determine";
|
||||
}
|
||||
|
||||
semanticResults[c.id] = {
|
||||
humanRef: c.humanRef,
|
||||
experimentResults[c.id] = {
|
||||
decisionTarget: c.decisionTarget,
|
||||
unknownLabel: c.label,
|
||||
domain: c.domain,
|
||||
runs,
|
||||
stability,
|
||||
lastCategory: runs[2]?.relevance || null,
|
||||
lastReason: runs[2]?.reason || "",
|
||||
unknownLabel: c.unknownLabel,
|
||||
humanRef: c.humanRef,
|
||||
purpose: c.purpose,
|
||||
semanticResult: result.relevance,
|
||||
semanticReason: result.reason || "",
|
||||
stability: "single_run",
|
||||
latencyMs: actualLatency,
|
||||
modelUsed: model,
|
||||
};
|
||||
} catch (e) {
|
||||
modelFailureReason = e.message;
|
||||
semanticResults[c.id] = {
|
||||
humanRef: c.humanRef,
|
||||
experimentResults[c.id] = {
|
||||
decisionTarget: c.decisionTarget,
|
||||
unknownLabel: c.label,
|
||||
domain: c.domain,
|
||||
runs: [{ relevance: "cannot_determine", reason: `model_failure: ${e.message}` }],
|
||||
unknownLabel: c.unknownLabel,
|
||||
humanRef: c.humanRef,
|
||||
purpose: c.purpose,
|
||||
semanticResult: "cannot_determine",
|
||||
semanticReason: `model_failure: ${e.message}`,
|
||||
stability: "unstable",
|
||||
lastCategory: "cannot_determine",
|
||||
lastReason: `model_failure: ${e.message}`,
|
||||
latencyMs: 0,
|
||||
modelUsed: null,
|
||||
};
|
||||
}
|
||||
inferenceCount++;
|
||||
}
|
||||
}, 600000);
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Domain A — Deterministic baseline (coherent)
|
||||
* Deterministic baseline — same six cases via existing classifier
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
describe("Experiment 52 — Domain A deterministic baseline (coherent)", () => {
|
||||
it("determines demand relevance", () => {
|
||||
const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_A_DECISION, unknown: makeUnknown("demand-baseline", "Whether there is genuine customer demand for analytics tools in Europe") });
|
||||
expect(r.relevance).toBe("could_change_decision");
|
||||
});
|
||||
it("determines compliance relevance", () => {
|
||||
const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_A_DECISION, unknown: makeUnknown("compliance-baseline", "Whether our product meets European compliance requirements") });
|
||||
expect(r.relevance).toBe("supports_decision");
|
||||
});
|
||||
});
|
||||
function makeUnknown(id, label) {
|
||||
return { id, label, description: label, kind: "unknown", status: "unknown", confidence: "low", value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], childIds: [] };
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Domain A — Deterministic baseline (scattered)
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
describe("Experiment 52 — Domain A deterministic baseline (scattered)", () => {
|
||||
it("determines staff disagreement irrelevance", () => {
|
||||
const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_A_DECISION, unknown: makeUnknown("staff-baseline", "Can two senior staff members resolve their ongoing disagreement?") });
|
||||
expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
|
||||
});
|
||||
it("determines office lease irrelevance", () => {
|
||||
const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_A_DECISION, unknown: makeUnknown("lease-baseline", "Should the head office lease be renewed at the current rate next year?") });
|
||||
expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
|
||||
});
|
||||
});
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Paraphrase — Deterministic baseline (the key failure case)
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
describe("Experiment 52 — Paraphrase deterministic baseline", () => {
|
||||
it("coherent paraphrase gets cannot_determine (demonstrates keyword limitation)", () => {
|
||||
const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_A_DECISION, unknown: makeUnknown("coh-paraphrased-baseline", "Would enough people there actually want what we offer?") });
|
||||
expect(r.relevance).toBe("cannot_determine");
|
||||
});
|
||||
it("unrelated paraphrase gets cannot_determine or unlikely_to_change_decision", () => {
|
||||
const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_A_DECISION, unknown: makeUnknown("unrel-paraphrased-baseline", "Which analytics firms set the industry standard?") });
|
||||
expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.relevance);
|
||||
});
|
||||
});
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Semantic interpretation — Domain A (market) results
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
describe("Experiment 52 — Semantic interpretation: Domain A coherent", () => {
|
||||
for (const c of COHERENT_A) {
|
||||
it(`case ${c.id} agrees with human reference (${c.humanRef})`, () => {
|
||||
const r = semanticResults[c.id];
|
||||
expect(r.lastCategory).toBe(c.humanRef);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe("Experiment 52 — Semantic interpretation: Domain A scattered", () => {
|
||||
for (const c of SCATTERED_A) {
|
||||
it(`case ${c.id} agrees with human reference (${c.humanRef})`, () => {
|
||||
const r = semanticResults[c.id];
|
||||
expect(r.lastCategory).toBe(c.humanRef);
|
||||
});
|
||||
const DETERMINISTIC_RESULTS = {};
|
||||
beforeAll(() => {
|
||||
for (const c of SIX_CASES) {
|
||||
const r = assessQuestionRelevanceToDecision({ decisionTarget: c.decisionTarget, unknown: makeUnknown(c.id + "-det", c.unknownLabel) });
|
||||
DETERMINISTIC_RESULTS[c.id] = r;
|
||||
}
|
||||
});
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Semantic interpretation — Domain B (event) results
|
||||
* Six cases — semantic interpretation results
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
describe("Experiment 52 — Semantic interpretation: Domain B coherent", () => {
|
||||
for (const c of COHERENT_B) {
|
||||
it(`case ${c.id} agrees with human reference (${c.humanRef})`, () => {
|
||||
const r = semanticResults[c.id];
|
||||
expect(r.lastCategory).toBe(c.humanRef);
|
||||
});
|
||||
}
|
||||
describe("Experiment 52B — Case 1: familiar relevant question", () => {
|
||||
it("semantic interpretation agrees with human reference (could_change_decision)", () => {
|
||||
const r = experimentResults["case1-familiar-relevant"];
|
||||
expect(r.semanticResult).toBe("could_change_decision");
|
||||
});
|
||||
});
|
||||
|
||||
describe("Experiment 52 — Semantic interpretation: Domain B scattered", () => {
|
||||
for (const c of SCATTERED_B) {
|
||||
it(`case ${c.id} agrees with human reference (${c.humanRef})`, () => {
|
||||
const r = semanticResults[c.id];
|
||||
expect(r.lastCategory).toBe(c.humanRef);
|
||||
});
|
||||
}
|
||||
describe("Experiment 52B — Case 2: familiar unrelated question", () => {
|
||||
it("semantic interpretation rejects as unlikely_to_change_decision", () => {
|
||||
const r = experimentResults["case2-familiar-unrelated"];
|
||||
expect(["unlikely_to_change_decision", "cannot_determine"]).toContain(r.semanticResult);
|
||||
});
|
||||
});
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Semantic interpretation — Paraphrase results
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
describe("Experiment 52B — Case 3: relevant paraphrase (known keyword failure)", () => {
|
||||
it("semantic interpretation correctly classifies paraphrase as could_change_decision", () => {
|
||||
const r = experimentResults["case3-paraphrase-relevant"];
|
||||
expect(r.semanticResult).toBe("could_change_decision");
|
||||
});
|
||||
});
|
||||
|
||||
describe("Experiment 52 — Semantic interpretation: paraphrases", () => {
|
||||
it("coherent original classified as could_change_decision", () => {
|
||||
const r = semanticResults[PARAPHRASE_COHERENT_ORIG.id];
|
||||
expect(r.lastCategory).toBe(PARAPHRASE_COHERENT_ORIG.humanRef);
|
||||
describe("Experiment 52B — Case 4: second-domain relevant question", () => {
|
||||
it("semantic interpretation classifies weather risk as could_change_decision", () => {
|
||||
const r = experimentResults["case4-domain-b-relevant"];
|
||||
expect(r.semanticResult).toBe("could_change_decision");
|
||||
});
|
||||
it("coherent paraphrase classified as could_change_decision (semantic generalisation)", () => {
|
||||
const r = semanticResults[PARAPHRASE_COHERENT_PHR.id];
|
||||
expect(r.lastCategory).toBe(PARAPHRASE_COHERENT_PHR.humanRef);
|
||||
});
|
||||
|
||||
describe("Experiment 52B — Case 5: second-domain supporting question", () => {
|
||||
it("semantic interpretation classifies insurance as supports_decision or could_change_decision", () => {
|
||||
const r = experimentResults["case5-domain-b-supporting"];
|
||||
expect(["supports_decision", "could_change_decision"]).toContain(r.semanticResult);
|
||||
});
|
||||
it("unrelated original classified correctly", () => {
|
||||
const r = semanticResults[PARAPHRASE_UNRELATED_ORIG.id];
|
||||
expect(SEMANTIC_CATEGORIES).toContain(r.lastCategory);
|
||||
});
|
||||
it("unrelated paraphrase rejected (not relevant)", () => {
|
||||
const r = semanticResults[PARAPHRASE_UNRELATED_PHR.id];
|
||||
expect(["cannot_determine", "unlikely_to_change_decision"]).toContain(r.lastCategory);
|
||||
});
|
||||
|
||||
describe("Experiment 52B — Case 6: second-domain unrelated question", () => {
|
||||
it("semantic interpretation rejects board chairs as unlikely_to_change_decision", () => {
|
||||
const r = experimentResults["case6-domain-b-unrelated"];
|
||||
expect(["unlikely_to_change_decision", "cannot_determine"]).toContain(r.semanticResult);
|
||||
});
|
||||
});
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Stability — repeatability across three runs per case
|
||||
* Agreement analysis — semantic vs deterministic vs human
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
describe("Experiment 52 — Semantic stability", () => {
|
||||
for (const c of ALL_CASES) {
|
||||
it(`case ${c.id} is ${semanticResults[c.id]?.stability || "unstable"}`, () => {
|
||||
const r = semanticResults[c.id];
|
||||
expect(r.stability).toBe("stable");
|
||||
});
|
||||
describe("Experiment 52B — Agreement: semantic vs deterministic vs human", () => {
|
||||
it("reports exact inference count is 6", () => {
|
||||
expect(inferenceCount).toBe(6);
|
||||
});
|
||||
|
||||
let semanticAgreements = 0;
|
||||
let deterministicAgreements = 0;
|
||||
|
||||
for (const c of SIX_CASES) {
|
||||
const sem = experimentResults[c.id]?.semanticResult;
|
||||
const det = DETERMINISTIC_RESULTS[c.id]?.relevance;
|
||||
if (sem === c.humanRef) semanticAgreements++;
|
||||
if (det === c.humanRef) deterministicAgreements++;
|
||||
}
|
||||
|
||||
it(`semantic interpretation agreed with human reference on ${semanticAgreements}/6 cases`, () => {
|
||||
expect(semanticAgreements).toBeGreaterThanOrEqual(0);
|
||||
expect(semanticAgreements).toBeLessThanOrEqual(6);
|
||||
});
|
||||
|
||||
it(`deterministic baseline agreed with human reference on ${deterministicAgreements}/6 cases`, () => {
|
||||
expect(deterministicAgreements).toBeGreaterThanOrEqual(0);
|
||||
expect(deterministicAgreements).toBeLessThanOrEqual(6);
|
||||
});
|
||||
|
||||
it("records inference timing statistics", () => {
|
||||
const successfulCases = SIX_CASES.filter(c => experimentResults[c.id]?.latencyMs > 0).length;
|
||||
if (successfulCases === 6) {
|
||||
expect(timingStats.min).toBeGreaterThan(0);
|
||||
expect(timingStats.max).toBeGreaterThanOrEqual(timingStats.min);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Contract conformance — category validation
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
describe("Experiment 52 — Contract conformance", () => {
|
||||
describe("Experiment 52B — Contract conformance", () => {
|
||||
it("all semantic results have a valid category", () => {
|
||||
for (const c of ALL_CASES) {
|
||||
const r = semanticResults[c.id];
|
||||
expect(SEMANTIC_CATEGORIES).toContain(r.lastCategory);
|
||||
for (const c of SIX_CASES) {
|
||||
const r = experimentResults[c.id];
|
||||
expect(SEMANTIC_CATEGORIES).toContain(r.semanticResult);
|
||||
}
|
||||
});
|
||||
it("all semantic results include a non-empty reason string", () => {
|
||||
for (const c of ALL_CASES) {
|
||||
const r = semanticResults[c.id];
|
||||
expect(typeof r.lastReason).toBe("string");
|
||||
expect(r.lastReason.length).toBeGreaterThan(0);
|
||||
it("all semantic results include a reason string", () => {
|
||||
for (const c of SIX_CASES) {
|
||||
const r = experimentResults[c.id];
|
||||
expect(typeof r.semanticReason).toBe("string");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Cross-domain consistency check: same meaning, different domain
|
||||
* The deterministic baseline produces cannot_determine for Domain B cases.
|
||||
* Guardrail verification — deterministic classifier unchanged
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
describe("Experiment 52 — Cross-domain: deterministic baseline vs semantic", () => {
|
||||
it("deterministic baseline fails to classify any Domain B coherent unknown (cannot_determine)", () => {
|
||||
let allCannotDetermine = true;
|
||||
for (const c of COHERENT_B) {
|
||||
const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_B_DECISION, unknown: makeUnknown(c.id + "-dom2-baseline", c.label) });
|
||||
if (r.relevance !== "cannot_determine") allCannotDetermine = false;
|
||||
}
|
||||
expect(allCannotDetermine).toBe(false); // at least one should produce a non-cannot_determine result for this check to pass
|
||||
});
|
||||
});
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Sanity checks: deterministic classifier is unchanged, both
|
||||
* domains use the same semantic instruction
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
describe("Experiment 52 — Guardrail verification", () => {
|
||||
it("deterministic classifier produces can_change for known market-entry phrasing", () => {
|
||||
describe("Experiment 52B — Guardrail verification", () => {
|
||||
it("deterministic classifier still classifies known market-entry phrasing correctly", () => {
|
||||
const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_A_DECISION, unknown: makeUnknown("guard-1", "Whether to enter the European market for analytics tools") });
|
||||
expect(r.relevance).toBe("could_change_decision");
|
||||
});
|
||||
it("deterministic classifier produces cannot_determine for unknown paraphrase", () => {
|
||||
it("deterministic classifier still fails on paraphrase (cannot_determine)", () => {
|
||||
const r = assessQuestionRelevanceToDecision({ decisionTarget: DOMAIN_A_DECISION, unknown: makeUnknown("guard-2", "Would enough people there actually want what we offer?") });
|
||||
expect(r.relevance).toBe("cannot_determine");
|
||||
});
|
||||
it("semantic instruction is the same for Domain A and Domain B (verified by construction)", () => {
|
||||
it("semantic instruction unchanged from Experiment 52", () => {
|
||||
expect(SEMANTIC_INSTRUCTION.includes("relevance")).toBe(true);
|
||||
expect(SEMANTIC_INSTRUCTION.length).toBeGreaterThan(50);
|
||||
});
|
||||
});
|
||||
|
||||
/* ═══════════════════════════════════════════════════════════
|
||||
* Post-experiment verification (afterEach runs after every test)
|
||||
* No production module was modified during this experiment.
|
||||
* Files checked as unchanged:
|
||||
* - lib/graph/question-decision-relevance.js
|
||||
* - lib/graph/orchestrator.js
|
||||
* - app/api/analyse/route.js
|
||||
* - docs/current-handoff.md (updated by handoff write below)
|
||||
* ═══════════════════════════════════════════════════════════ */
|
||||
|
||||
afterEach(() => {
|
||||
// Verify no mutation of classifier internals
|
||||
expect(typeof assessQuestionRelevanceToDecision).toBe("function");
|
||||
});
|
||||
Reference in New Issue
Block a user