354 lines
16 KiB
JavaScript
354 lines
16 KiB
JavaScript
import { describe, it, expect } from "vitest";
|
||
import { config } from "dotenv";
|
||
import path from "path";
|
||
import { fileURLToPath } from "url";
|
||
|
||
// Load project .env.local — same source as production
|
||
const __filename = fileURLToPath(import.meta.url);
|
||
const __dirname = path.dirname(__filename);
|
||
config({ path: path.resolve(__dirname, "../../.env.local") });
|
||
|
||
const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL;
|
||
const OLLAMA_MODEL = process.env.OLLAMA_MODEL;
|
||
|
||
if (!OLLAMA_BASE_URL || !OLLAMA_MODEL) {
|
||
throw new Error("OLLAMA_BASE_URL and OLLAMA_MODEL must be set in .env.local");
|
||
}
|
||
|
||
// ──────────────────────────────────────────────
|
||
// Semantic instruction — identical to Experiment 54K (unchanged)
|
||
// ──────────────────────────────────────────────
|
||
|
||
const SEMANTIC_INSTRUCTION = `
|
||
Compare the interpretation with the exact source text. Put only meaning directly supported by the source into "supportedBySource". Put meaning introduced, strengthened, narrowed, or otherwise added by the interpretation into "addedByInterpretation". Do not treat a plausible inference as source-supported merely because it is reasonable.
|
||
|
||
Return valid JSON only in this shape:
|
||
{
|
||
"supportedBySource": ["short factual statements"],
|
||
"addedByInterpretation": ["short factual statements"]
|
||
}
|
||
`;
|
||
|
||
/**
|
||
* Make one live Ollama chat call — identical pattern to Experiment 54K.
|
||
*/
|
||
async function callSemanticModel(source, interpretation) {
|
||
const messages = [
|
||
{ role: "system", content: SEMANTIC_INSTRUCTION.trim() },
|
||
{ role: "user", content: `Source: "${source}"\nInterpretation: "${interpretation}"` },
|
||
];
|
||
|
||
const res = await fetch(`${OLLAMA_BASE_URL}/api/chat`, {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
model: OLLAMA_MODEL,
|
||
messages,
|
||
format: "json",
|
||
stream: false,
|
||
}),
|
||
});
|
||
|
||
if (!res.ok) {
|
||
throw new Error(`Ollama API error: ${res.status} ${res.statusText}`);
|
||
}
|
||
|
||
const data = await res.json();
|
||
const rawContent = data.message?.content ?? "";
|
||
|
||
// Strip markdown code fences if present
|
||
const cleaned = rawContent.replace(/```(?:json)?\s*/g, "").replace(/```\s*/g, "");
|
||
|
||
return JSON.parse(cleaned.trim());
|
||
}
|
||
|
||
// ──────────────────────────────────────────────
|
||
// Fixed human references — identical to Experiment 54K
|
||
// ──────────────────────────────────────────────
|
||
|
||
const FIXED_CASE_A = {
|
||
id: "Case A — Strengthening Control",
|
||
source: "Revenue is down. I think pricing may be part of the problem, but I am not sure.",
|
||
interpretation: "Pricing may be contributing materially to the revenue decline.",
|
||
refSupported: [
|
||
{ check: (s) => s.toLowerCase().includes("revenue") && (s.toLowerCase().includes("down") || s.toLowerCase().includes("decline")), label: "revenue is down" },
|
||
{ check: (s) => s.toLowerCase().includes("pricing") && (s.toLowerCase().includes("part") || s.toLowerCase().includes("factor") || s.toLowerCase().includes("cause")), label: "pricing may be part of the problem" },
|
||
{ check: (s) => s.toLowerCase().includes("unsure") || s.toLowerCase().includes("uncertain") || s.toLowerCase().includes("not sure") || s.toLowerCase().includes("uncertainty"), label: "user uncertainty" },
|
||
],
|
||
refAdded: [
|
||
{ check: (s) => s.toLowerCase().includes("materially") || s.toLowerCase().includes("significant impact") || s.toLowerCase().includes("substantial"), label: "stronger/material impact from pricing" },
|
||
],
|
||
};
|
||
|
||
const FIXED_CASE_B = {
|
||
id: "Case B — Multi-Addition",
|
||
source: "Revenue is down. I think pricing may be part of the problem, but I am not sure.",
|
||
interpretation: "The revenue decline may have causes other than pricing, and pricing has not yet been established as the main problem.",
|
||
refSupported: [
|
||
{ check: (s) => s.toLowerCase().includes("revenue") && (s.toLowerCase().includes("down") || s.toLowerCase().includes("decline")), label: "revenue is down" },
|
||
{ check: (s) => s.toLowerCase().includes("pricing") && (s.toLowerCase().includes("part") || s.toLowerCase().includes("factor") || s.toLowerCase().includes("cause")), label: "pricing may be part of the problem" },
|
||
{ check: (s) => s.toLowerCase().includes("unsure") || s.toLowerCase().includes("uncertain") || s.toLowerCase().includes("not sure") || s.toLowerCase().includes("uncertainty"), label: "user uncertainty" },
|
||
],
|
||
refAdded: [
|
||
{ check: (s) => s.toLowerCase().includes("other") || s.toLowerCase().includes("alternative") || s.toLowerCase().includes("besides"), label: "causes other than pricing may exist" },
|
||
{ check: (s) => /not\s+.*established/i.test(s) || s.toLowerCase().includes("confirmed as primary") || /not.*main problem/i.test(s) || /not\s+.*primary/i.test(s), label: "pricing framed as not established as main problem" },
|
||
],
|
||
};
|
||
|
||
// ──────────────────────────────────────────────
|
||
// Stability evaluation — classification per run only
|
||
// ──────────────────────────────────────────────
|
||
|
||
function evaluateGrounding(runResult, reference) {
|
||
const supported = (runResult.supportedBySource ?? []).map((s) => s.trim());
|
||
const added = (runResult.addedByInterpretation ?? []).map((s) => s.trim());
|
||
|
||
let criticalIssues = [];
|
||
let minorIssues = [];
|
||
|
||
// Check: all reference-supported concepts present?
|
||
for (const concept of reference.refSupported) {
|
||
const found = supported.some(concept.check);
|
||
if (!found) {
|
||
criticalIssues.push(`missing supported: ${concept.label}`);
|
||
}
|
||
}
|
||
|
||
// Check: any reference-added concept incorrectly in supported? (leakage INTO supported)
|
||
for (const concept of reference.refAdded) {
|
||
const leaked = supported.some(concept.check);
|
||
if (leaked) {
|
||
criticalIssues.push(`leaked into supported: ${concept.label}`);
|
||
}
|
||
}
|
||
|
||
// Check: reference-added concepts present?
|
||
for (const concept of reference.refAdded) {
|
||
const found = added.some(concept.check);
|
||
if (!found) {
|
||
minorIssues.push(`missing added: ${concept.label}`);
|
||
}
|
||
}
|
||
|
||
// Classification logic
|
||
if (criticalIssues.length === 0 && minorIssues.length === 0) return "grounding_correct";
|
||
if (criticalIssues.some((i) => i.includes("leaked"))) {
|
||
return "grounding_failed";
|
||
}
|
||
if (criticalIssues.length >= 2) return "grounding_failed";
|
||
return "partial_grounding";
|
||
}
|
||
|
||
// ──────────────────────────────────────────────
|
||
// Test suite — 2 cases × 3 runs = 6 live inference calls
|
||
// ──────────────────────────────────────────────
|
||
|
||
const TEST_TIMEOUT_MS = 60_000;
|
||
|
||
describe("Experiment 54L — Semantic Grounding Stability (test-only)", () => {
|
||
const results = [];
|
||
const timings = [];
|
||
|
||
// Case A: Strengthening Control × 3 runs
|
||
for (let run = 1; run <= 3; run++) {
|
||
it(
|
||
`Case A run ${run} — strengthening control`,
|
||
async () => {
|
||
const t0 = performance.now();
|
||
const result = await callSemanticModel(
|
||
FIXED_CASE_A.source,
|
||
FIXED_CASE_A.interpretation,
|
||
);
|
||
const elapsed = performance.now() - t0;
|
||
timings.push(elapsed);
|
||
|
||
const classification = evaluateGrounding(result, FIXED_CASE_A);
|
||
|
||
results.push({
|
||
case: "A",
|
||
run,
|
||
source: FIXED_CASE_A.source,
|
||
interpretation: FIXED_CASE_A.interpretation,
|
||
semanticResult: result,
|
||
timingMs: Number(elapsed.toFixed(2)),
|
||
classification,
|
||
conceptCheck: {
|
||
refSupported: FIXED_CASE_A.refSupported.map((c) => ({
|
||
label: c.label,
|
||
presentInSupported: (result.supportedBySource ?? []).some(c.check),
|
||
presentInAdded: (result.addedByInterpretation ?? []).some(c.check),
|
||
})),
|
||
refAdded: FIXED_CASE_A.refAdded.map((c) => ({
|
||
label: c.label,
|
||
presentInSupported: (result.supportedBySource ?? []).some(c.check),
|
||
presentInAdded: (result.addedByInterpretation ?? []).some(c.check),
|
||
})),
|
||
},
|
||
});
|
||
|
||
console.log(
|
||
`\n=== Case A run ${run} (${classification}) — ${(elapsed / 1000).toFixed(1)}s ===`,
|
||
);
|
||
console.log(`supportedBySource:`, JSON.stringify(result.supportedBySource, null, 2));
|
||
console.log(`addedByInterpretation:`, JSON.stringify(result.addedByInterpretation, null, 2));
|
||
},
|
||
TEST_TIMEOUT_MS,
|
||
);
|
||
}
|
||
|
||
// Case B: Multi-Addition × 3 runs
|
||
for (let run = 1; run <= 3; run++) {
|
||
it(
|
||
`Case B run ${run} — multi-addition`,
|
||
async () => {
|
||
const t0 = performance.now();
|
||
const result = await callSemanticModel(
|
||
FIXED_CASE_B.source,
|
||
FIXED_CASE_B.interpretation,
|
||
);
|
||
const elapsed = performance.now() - t0;
|
||
timings.push(elapsed);
|
||
|
||
const classification = evaluateGrounding(result, FIXED_CASE_B);
|
||
|
||
results.push({
|
||
case: "B",
|
||
run,
|
||
source: FIXED_CASE_B.source,
|
||
interpretation: FIXED_CASE_B.interpretation,
|
||
semanticResult: result,
|
||
timingMs: Number(elapsed.toFixed(2)),
|
||
classification,
|
||
conceptCheck: {
|
||
refSupported: FIXED_CASE_B.refSupported.map((c) => ({
|
||
label: c.label,
|
||
presentInSupported: (result.supportedBySource ?? []).some(c.check),
|
||
presentInAdded: (result.addedByInterpretation ?? []).some(c.check),
|
||
})),
|
||
refAdded: FIXED_CASE_B.refAdded.map((c) => ({
|
||
label: c.label,
|
||
presentInSupported: (result.supportedBySource ?? []).some(c.check),
|
||
presentInAdded: (result.addedByInterpretation ?? []).some(c.check),
|
||
})),
|
||
},
|
||
});
|
||
|
||
console.log(
|
||
`\n=== Case B run ${run} (${classification}) — ${(elapsed / 1000).toFixed(1)}s ===`,
|
||
);
|
||
console.log(`supportedBySource:`, JSON.stringify(result.supportedBySource, null, 2));
|
||
console.log(`addedByInterpretation:`, JSON.stringify(result.addedByInterpretation, null, 2));
|
||
},
|
||
TEST_TIMEOUT_MS,
|
||
);
|
||
}
|
||
|
||
// ── Stability analysis and required questions ──
|
||
it(
|
||
"54L — stability analysis and required questions",
|
||
() => {
|
||
const caseARuns = results.filter((r) => r.case === "A");
|
||
const caseBRuns = results.filter((r) => r.case === "B");
|
||
|
||
expect(caseARuns.length).toBe(3);
|
||
expect(caseBRuns.length).toBe(3);
|
||
expect(results.length).toBe(6);
|
||
|
||
// Q1: Did materially strengthening remain outside supportedBySource?
|
||
const caseAMateriallyInSupported = caseARuns.some((run) =>
|
||
(run.semanticResult.supportedBySource ?? []).some((s) =>
|
||
s.toLowerCase().includes("materially") ||
|
||
s.toLowerCase().includes("significant impact") ||
|
||
s.toLowerCase().includes("substantial"),
|
||
),
|
||
);
|
||
|
||
// Q2: Was Case A consistent across runs?
|
||
const caseAClassifications = caseARuns.map((r) => r.classification);
|
||
const caseAStable = caseAClassifications.every((c) => c === caseAClassifications[0]);
|
||
|
||
// Q3-4: Case B concept detection counts
|
||
let otherCausesCount = 0;
|
||
let notEstablishedCount = 0;
|
||
|
||
for (const run of caseBRuns) {
|
||
const added = (run.semanticResult.addedByInterpretation ?? []).map((s) => s.toLowerCase());
|
||
if (added.some((s) => s.includes("other") || s.includes("alternative"))) otherCausesCount++;
|
||
if (added.some((s) => /not.*established/i.test(s) || /not.*main problem/i.test(s) || /not.*primary/i.test(s))) notEstablishedCount++;
|
||
}
|
||
|
||
// Q5-6: Leakage checks
|
||
const interpretationLeakedIntoSupported = results.some((r) =>
|
||
(r.conceptCheck.refAdded ?? []).some((c) => c.presentInSupported),
|
||
);
|
||
const sourceMovedToAdded = results.some((r) =>
|
||
(r.conceptCheck.refSupported ?? []).some((c) => c.presentInAdded),
|
||
);
|
||
|
||
// Q7-8: Material stability per case
|
||
const caseBClassifications = caseBRuns.map((r) => r.classification);
|
||
const caseBStable = caseBClassifications.every((c) => c === caseBClassifications[0]);
|
||
|
||
// Timing
|
||
const totalMs = timings.reduce((a, b) => a + b, 0);
|
||
const avgMs = totalMs / timings.length;
|
||
const fastestMs = Math.min(...timings);
|
||
const slowestMs = Math.max(...timings);
|
||
|
||
// Print results
|
||
console.log("\n=== Experiment 54L Summary ===");
|
||
console.log(`Total runs: ${results.length}`);
|
||
console.log(`Case A classifications:`, caseAClassifications.join(", "));
|
||
console.log(`Case B classifications:`, caseBClassifications.join(", "));
|
||
console.log(
|
||
`Q1 - Material strengthening in supportedBySource: ${caseAMateriallyInSupported ? "YES (LEAKAGE)" : "NO (correct)"}`,
|
||
);
|
||
console.log(`Q2 - Case A stable across runs: ${caseAStable}`);
|
||
console.log(`Q3 - "Other causes" detected: ${otherCausesCount}/3`);
|
||
console.log(`Q4 - "Not established as main problem" detected: ${notEstablishedCount}/3`);
|
||
console.log(
|
||
`Q5 - Interpretation-added leaked into supportedBySource: ${interpretationLeakedIntoSupported ? "YES" : "NO"}`,
|
||
);
|
||
console.log(
|
||
`Q6 - Source-supported moved to addedByInterpretation: ${sourceMovedToAdded ? "YES" : "NO"}`,
|
||
);
|
||
console.log(`Q7 - Case A materially stable: ${caseAStable}`);
|
||
console.log(`Q8 - Case B materially stable: ${caseBStable}`);
|
||
|
||
// Q9: Answer from data
|
||
const anyLeakage = caseARuns.some(
|
||
(r) => r.conceptCheck.refAdded.some((c) => c.presentInSupported),
|
||
);
|
||
const anyAdditionMissed = caseBRuns.some(
|
||
(r) => r.conceptCheck.refAdded.some((c) => !c.presentInAdded && !c.presentInSupported),
|
||
);
|
||
console.log(`Q9 - More stable about preventing leakage vs detecting additions: ${!anyLeakage && anyAdditionMissed ? "More stable about preventing leakage" : "Inconclusive from this data"}`);
|
||
console.log(`Q10 - Establishes which interpretation is better: No`);
|
||
console.log(`Q11 - Establishes downstream question: No`);
|
||
|
||
// Stability summary
|
||
console.log(`\nCase A material stability: ${caseAStable ? "stable" : "variable"}`);
|
||
console.log(`Case B material stability: ${caseBStable ? "stable" : "variable"}`);
|
||
|
||
// Overall conclusion selection
|
||
let conclusion;
|
||
if (timings.length < 6) {
|
||
conclusion = "Live probe could not be completed";
|
||
} else if (caseAStable && caseBStable && !interpretationLeakedIntoSupported) {
|
||
conclusion = "Semantic grounding is materially stable across the tested repeats";
|
||
} else if (!interpretationLeakedIntoSupported) {
|
||
conclusion = "Grounding boundary is stable but addition detection varies";
|
||
} else {
|
||
conclusion = "Semantic grounding varies materially across repeated identical inputs";
|
||
}
|
||
console.log(`\nConclusion: ${conclusion}`);
|
||
|
||
console.log(`\nTiming — Total: ${(totalMs / 1000).toFixed(1)}s, Avg: ${(avgMs / 1000).toFixed(1)}s, Fastest: ${(fastestMs / 1000).toFixed(1)}s, Slowest: ${(slowestMs / 1000).toFixed(1)}s`);
|
||
|
||
// Final assertion — always pass so summary is recorded
|
||
expect(results.length).toBe(6);
|
||
},
|
||
TEST_TIMEOUT_MS,
|
||
);
|
||
});
|