robbond
8d268efb12
- Add nullish coalescing on fix.findings ?? [] so absent findings array in a fixture is represented as zero Findings [] for the A/B input apparatus. - Append product-launch second controlled case (live-product-launch- start-response.json) with zero Findings to the epistemic comparison test suite. All 34 tests pass deterministically. - Update docs/current-handoff.md to record the zero-Findings case.
295 lines
15 KiB
JavaScript
295 lines
15 KiB
JavaScript
/**
|
|
* Deterministic control of report epistemic comparison inputs.
|
|
*
|
|
* Proves BEFORE any future model invocation that the A/B experiment
|
|
* inputs are correctly controlled: only unresolved-question visibility
|
|
* differs between A and B; everything else is identical.
|
|
*
|
|
* No live model calls. No mocking. Pure input-contract verification.
|
|
*/
|
|
|
|
import { describe, it, expect } from "vitest";
|
|
import fixture from "../fixtures/progressed-investigation-manufacturing.json";
|
|
import productLaunchFixture from "../fixtures/live-product-launch-start-response.json";
|
|
|
|
const FIXED_PLAUSIBLE =
|
|
"The rise in absolute complaint counts is likely an artifact of both the new CRM tagging system broadening what gets recorded as a complaint, combined with a proportional increase from higher production volume rather than a genuine deterioration in product quality.";
|
|
|
|
function buildABInputs(fix, includeUQ) {
|
|
const sg = fix.situationGraph;
|
|
const se = sg.nodes.filter(n => n.status === "known" || n.status === "supported").map(n => ({id: n.id, kind: n.kind, status: n.status, label: n.label}));
|
|
const fi = (fix.findings ?? []).filter(f2 => f2.status === "provisional" && f2.userDisposition === null).map(f3 => ({id: f3.id, proposition: f3.proposition}));
|
|
const pi = [FIXED_PLAUSIBLE];
|
|
const uq = sg.nodes.filter(n => n.kind === "unknown" && n.status === "unknown").map(n => ({id: n.id, label: n.label, description: n.description}));
|
|
const ss = {centralStatement: sg.centralStatement, activeUnknownNodeId: sg.activeUnknownNodeId, resolvedNodeIds: [...sg.resolvedNodeIds], nodeCount: sg.nodes.length, edgeCount: sg.edges.length};
|
|
const pc = {provider: "ollama", model: process.env.OLLAMA_MODEL || "configured-model"};
|
|
return {supportedEvidence: se, findings: fi, plausibleInterpretations: pi, unresolvedQuestions: includeUQ ? [...uq] : [], scenarioState: ss, providerConfig: pc};
|
|
}
|
|
|
|
function deepEqual(a, b) {
|
|
if (a === b) return true;
|
|
if (a == null || b == null) return false;
|
|
if (typeof a !== typeof b) return false;
|
|
if (Array.isArray(a)) { if (!Array.isArray(b) || a.length !== b.length) return false; for (let i = 0; i < a.length; i++) if (!deepEqual(a[i], b[i])) return false; return true; }
|
|
if (typeof a === "object") { const kA = Object.keys(a).sort(), kB = Object.keys(b).sort(); if (kA.length !== kB.length) return false; for (let i = 0; i < kA.length; i++) if (kA[i] !== kB[i] || !deepEqual(a[kA[i]], b[kB[i]])) return false; return true; }
|
|
return false;
|
|
}
|
|
|
|
describe("Controlled A/B report epistemic comparison inputs", () => {
|
|
it("builds canonical input objects for both A and B", () => {
|
|
const a = buildABInputs(fixture, false);
|
|
const b = buildABInputs(fixture, true);
|
|
["supportedEvidence","findings","plausibleInterpretations","unresolvedQuestions","scenarioState","providerConfig"].forEach(k => { expect(a).toHaveProperty(k); expect(b).toHaveProperty(k); });
|
|
});
|
|
|
|
it("A and B have identical admitted evidence", () => {
|
|
const a = buildABInputs(fixture, false); const b = buildABInputs(fixture, true);
|
|
expect(deepEqual(a.supportedEvidence, b.supportedEvidence)).toBe(true);
|
|
});
|
|
|
|
it("A and B have identical eligible Findings", () => {
|
|
const a = buildABInputs(fixture, false); const b = buildABInputs(fixture, true);
|
|
expect(deepEqual(a.findings, b.findings)).toBe(true);
|
|
});
|
|
|
|
it("A and B have identical fixed plausibleInterpretations", () => {
|
|
const a = buildABInputs(fixture, false); const b = buildABInputs(fixture, true);
|
|
expect(deepEqual(a.plausibleInterpretations, b.plausibleInterpretations)).toBe(true);
|
|
});
|
|
|
|
it("fixed plausible interpretation matches canonical value exactly", () => {
|
|
const a = buildABInputs(fixture, false);
|
|
expect(a.plausibleInterpretations).toHaveLength(1);
|
|
expect(a.plausibleInterpretations[0]).toBe(FIXED_PLAUSIBLE);
|
|
});
|
|
|
|
it("A and B reference the same progressed investigation state", () => {
|
|
const a = buildABInputs(fixture, false); const b = buildABInputs(fixture, true);
|
|
expect(deepEqual(a.scenarioState, b.scenarioState)).toBe(true);
|
|
});
|
|
|
|
it("provider/model configuration is identical for A and B", () => {
|
|
const a = buildABInputs(fixture, false); const b = buildABInputs(fixture, true);
|
|
expect(a.providerConfig.provider).toBe(b.providerConfig.provider);
|
|
expect(a.providerConfig.model).toBe(b.providerConfig.model);
|
|
});
|
|
|
|
it("A does not expose unresolved Open Questions (hidden)", () => {
|
|
const a = buildABInputs(fixture, false);
|
|
expect(a.unresolvedQuestions).toHaveLength(0);
|
|
});
|
|
|
|
it("B exposes exactly the canonical unresolved Open Questions", () => {
|
|
const b = buildABInputs(fixture, true);
|
|
const fu = fixture.situationGraph.nodes.filter(n => n.kind === "unknown" && n.status === "unknown");
|
|
expect(b.unresolvedQuestions).toHaveLength(fu.length);
|
|
for (const node of fu) { const found = b.unresolvedQuestions.find(uq => uq.id === node.id); expect(found).toBeDefined(); expect(found.label).toBe(node.label); expect(found.description).toBe(node.description); }
|
|
});
|
|
|
|
it("only unresolved visibility differs between A and B", () => {
|
|
const a = buildABInputs(fixture, false); const b = buildABInputs(fixture, true);
|
|
expect(deepEqual(a.supportedEvidence, b.supportedEvidence)).toBe(true);
|
|
expect(deepEqual(a.findings, b.findings)).toBe(true);
|
|
expect(deepEqual(a.plausibleInterpretations, b.plausibleInterpretations)).toBe(true);
|
|
expect(deepEqual(a.scenarioState, b.scenarioState)).toBe(true);
|
|
expect(deepEqual(a.providerConfig, b.providerConfig)).toBe(true);
|
|
expect(a.unresolvedQuestions).not.toEqual(b.unresolvedQuestions);
|
|
});
|
|
|
|
it("fixed plausible interpretation is not admitted evidence", () => {
|
|
const a = buildABInputs(fixture, false);
|
|
for (const e of a.supportedEvidence) { expect(e.label).not.toContain(FIXED_PLAUSIBLE); expect(["known","supported"]).toContain(e.status); }
|
|
});
|
|
|
|
it("unresolved questions are not admitted evidence", () => {
|
|
const b = buildABInputs(fixture, true);
|
|
for (const uq of b.unresolvedQuestions) { const fn = fixture.situationGraph.nodes.find(n => n.id === uq.id); expect(fn.status).toBe("unknown"); expect(fn.kind).toBe("unknown"); }
|
|
});
|
|
|
|
it("unresolved questions are not plausibleInterpretations", () => {
|
|
const b = buildABInputs(fixture, true);
|
|
for (const uq of b.unresolvedQuestions) { expect(b.plausibleInterpretations).not.toContain(uq.label); expect(b.plausibleInterpretations[0]).toBe(FIXED_PLAUSIBLE); }
|
|
});
|
|
|
|
it("no fixture node conflates unresolved and evidence categories", () => {
|
|
for (const node of fixture.situationGraph.nodes) {
|
|
if (node.kind === "unknown" && node.status === "unknown") expect(["known","supported"]).not.toContain(node.status);
|
|
else if (["known","supported"].includes(node.status)) expect(node.kind).not.toBe("unknown");
|
|
}
|
|
});
|
|
|
|
it("inputs are constructed purely from fixture without provider or model dependency", () => {
|
|
const a = buildABInputs(fixture, false); const b = buildABInputs(fixture, true);
|
|
expect(typeof a.supportedEvidence).toBe("object");
|
|
expect(typeof a.findings).toBe("object");
|
|
expect(typeof a.plausibleInterpretations).toBe("object");
|
|
expect(typeof a.unresolvedQuestions).toBe("object");
|
|
});
|
|
|
|
// ── Prompt-builder identity proof (A/B produce identical SECTION A) ────
|
|
|
|
it("buildOverviewSynthesisPrompt produces identical prompt text when unknown nodes are present vs removed", async () => {
|
|
const { buildOverviewSynthesisPrompt } = await import(
|
|
"@/lib/graph/investigation-overview-synthesis.js"
|
|
);
|
|
const aInputs = buildABInputs(fixture, false);
|
|
|
|
const baseSg = fixture.situationGraph;
|
|
const knownSupportedNodes = baseSg.nodes.filter(
|
|
(n) => ["known", "supported"].includes(n.status)
|
|
);
|
|
|
|
// Version A: filtered graph (unknowns removed)
|
|
const promptA = buildOverviewSynthesisPrompt(
|
|
{ ...baseSg, nodes: knownSupportedNodes },
|
|
aInputs.supportedEvidence,
|
|
aInputs.findings,
|
|
aInputs.plausibleInterpretations
|
|
);
|
|
|
|
// Version B: full graph (unknowns included) — should be identical because
|
|
// buildOverviewSynthesisPrompt already filters to known+supported only
|
|
const promptB = buildOverviewSynthesisPrompt(
|
|
baseSg,
|
|
aInputs.supportedEvidence,
|
|
aInputs.findings,
|
|
aInputs.plausibleInterpretations
|
|
);
|
|
|
|
expect(promptA).toBe(promptB);
|
|
});
|
|
|
|
it("fixed plausible interpretation carries NON-EVIDENCE epistemic status", () => {
|
|
const a = buildABInputs(fixture, false);
|
|
expect(a.plausibleInterpretations).toHaveLength(1);
|
|
expect(a.plausibleInterpretations[0]).toBe(FIXED_PLAUSIBLE);
|
|
const el = a.supportedEvidence.map(e => e.label.toLowerCase());
|
|
const il = FIXED_PLAUSIBLE.toLowerCase();
|
|
for (const label of el) expect(label).not.toContain(il);
|
|
});
|
|
|
|
it("fixture has the minimum required content for A/B comparison", () => {
|
|
const sg = fixture.situationGraph;
|
|
expect(sg.nodes.filter(n => ["known","supported"].includes(n.status)).length).toBeGreaterThan(0);
|
|
expect(sg.nodes.filter(n => n.kind === "unknown" && n.status === "unknown").length).toBeGreaterThan(0);
|
|
expect(fixture.findings.filter(f => f.status === "provisional" && f.userDisposition === null).length).toBeGreaterThan(0);
|
|
});
|
|
});
|
|
|
|
// ── Second case: product-launch (zero Findings) ───────────────────────
|
|
|
|
const PLAUSIBLE_PRODUCT_LAUNCH =
|
|
"The large enterprise customer's decision timeline alone does not explain launch timing; competitor activity and the £300k support cost create an independent commercial uncertainty that is unresolved and evidence-free.";
|
|
|
|
describe("Controlled A/B report epistemic comparison inputs — product-launch zero-Findings case", () => {
|
|
const pl = productLaunchFixture;
|
|
|
|
it("fixture has no findings array (legitimate zero-Findings investigation)", () => {
|
|
expect(pl.findings).toBeUndefined();
|
|
});
|
|
|
|
it("zero Findings represented as [] by apparatus", () => {
|
|
const a = buildABInputsForProductLaunch(pl, false);
|
|
const b = buildABInputsForProductLaunch(pl, true);
|
|
expect(a.findings).toStrictEqual([]);
|
|
expect(b.findings).toStrictEqual([]);
|
|
});
|
|
|
|
it("supported evidence present from graph", () => {
|
|
const a = buildABInputsForProductLaunch(pl, false);
|
|
expect(a.supportedEvidence.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("unresolved questions present from graph", () => {
|
|
const b = buildABInputsForProductLaunch(pl, true);
|
|
const fu = pl.situationGraph.nodes.filter(n => n.kind === "unknown" && n.status === "unknown");
|
|
expect(b.unresolvedQuestions).toHaveLength(fu.length);
|
|
expect(fu.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
it("A and B have identical admitted evidence", () => {
|
|
const a = buildABInputsForProductLaunch(pl, false); const b = buildABInputsForProductLaunch(pl, true);
|
|
expect(deepEqual(a.supportedEvidence, b.supportedEvidence)).toBe(true);
|
|
});
|
|
|
|
it("A and B have identical (zero) Findings", () => {
|
|
const a = buildABInputsForProductLaunch(pl, false); const b = buildABInputsForProductLaunch(pl, true);
|
|
expect(deepEqual(a.findings, b.findings)).toBe(true);
|
|
expect(a.findings).toStrictEqual([]);
|
|
});
|
|
|
|
it("A and B have identical fixed plausibleInterpretations", () => {
|
|
const a = buildABInputsForProductLaunch(pl, false); const b = buildABInputsForProductLaunch(pl, true);
|
|
expect(deepEqual(a.plausibleInterpretations, b.plausibleInterpretations)).toBe(true);
|
|
});
|
|
|
|
it("A and B reference the same investigation state", () => {
|
|
const a = buildABInputsForProductLaunch(pl, false); const b = buildABInputsForProductLaunch(pl, true);
|
|
expect(deepEqual(a.scenarioState, b.scenarioState)).toBe(true);
|
|
});
|
|
|
|
it("provider/model configuration is identical for A and B", () => {
|
|
const a = buildABInputsForProductLaunch(pl, false); const b = buildABInputsForProductLaunch(pl, true);
|
|
expect(a.providerConfig.provider).toBe(b.providerConfig.provider);
|
|
expect(a.providerConfig.model).toBe(b.providerConfig.model);
|
|
});
|
|
|
|
it("A does not expose unresolved Open Questions (hidden)", () => {
|
|
const a = buildABInputsForProductLaunch(pl, false);
|
|
expect(a.unresolvedQuestions).toHaveLength(0);
|
|
});
|
|
|
|
it("B exposes exactly the canonical unresolved Open Questions", () => {
|
|
const b = buildABInputsForProductLaunch(pl, true);
|
|
const fu = pl.situationGraph.nodes.filter(n => n.kind === "unknown" && n.status === "unknown");
|
|
expect(b.unresolvedQuestions).toHaveLength(fu.length);
|
|
});
|
|
|
|
it("only unresolved visibility differs between A and B", () => {
|
|
const a = buildABInputsForProductLaunch(pl, false); const b = buildABInputsForProductLaunch(pl, true);
|
|
expect(deepEqual(a.supportedEvidence, b.supportedEvidence)).toBe(true);
|
|
expect(deepEqual(a.findings, b.findings)).toBe(true);
|
|
expect(deepEqual(a.plausibleInterpretations, b.plausibleInterpretations)).toBe(true);
|
|
expect(deepEqual(a.scenarioState, b.scenarioState)).toBe(true);
|
|
expect(deepEqual(a.providerConfig, b.providerConfig)).toBe(true);
|
|
expect(a.unresolvedQuestions).not.toEqual(b.unresolvedQuestions);
|
|
});
|
|
|
|
it("A and B have identical input objects with all canonical keys", () => {
|
|
const a = buildABInputsForProductLaunch(pl, false);
|
|
const b = buildABInputsForProductLaunch(pl, true);
|
|
["supportedEvidence","findings","plausibleInterpretations","unresolvedQuestions","scenarioState","providerConfig"].forEach(k => { expect(a).toHaveProperty(k); expect(b).toHaveProperty(k); });
|
|
});
|
|
|
|
it("inputs are constructed purely from fixture without provider or model dependency", () => {
|
|
const a = buildABInputsForProductLaunch(pl, false); const b = buildABInputsForProductLaunch(pl, true);
|
|
expect(typeof a.supportedEvidence).toBe("object");
|
|
expect(typeof a.findings).toBe("object");
|
|
expect(typeof a.plausibleInterpretations).toBe("object");
|
|
expect(typeof a.unresolvedQuestions).toBe("object");
|
|
});
|
|
|
|
it("plausible interpretation is not invented from scenario text", () => {
|
|
const a = buildABInputsForProductLaunch(pl, false);
|
|
expect(a.plausibleInterpretations).toHaveLength(1);
|
|
// The fixed plausible is a deterministic constant, not derived from the fixture
|
|
expect(a.plausibleInterpretations[0]).toBe(PLAUSIBLE_PRODUCT_LAUNCH);
|
|
});
|
|
|
|
it("fixture situationGraph has both supported evidence and unknown nodes", () => {
|
|
const sg = pl.situationGraph;
|
|
expect(sg.nodes.filter(n => ["known","supported"].includes(n.status)).length).toBeGreaterThan(0);
|
|
expect(sg.nodes.filter(n => n.kind === "unknown" && n.status === "unknown").length).toBeGreaterThan(0);
|
|
});
|
|
});
|
|
|
|
function buildABInputsForProductLaunch(fix, includeUQ) {
|
|
const sg = fix.situationGraph;
|
|
const se = sg.nodes.filter(n => n.status === "known" || n.status === "supported").map(n => ({id: n.id, kind: n.kind, status: n.status, label: n.label}));
|
|
const fi = (fix.findings ?? []).filter(f2 => f2.status === "provisional" && f2.userDisposition === null).map(f3 => ({id: f3.id, proposition: f3.proposition}));
|
|
const pi = [PLAUSIBLE_PRODUCT_LAUNCH];
|
|
const uq = sg.nodes.filter(n => n.kind === "unknown" && n.status === "unknown").map(n => ({id: n.id, label: n.label, description: n.description}));
|
|
const ss = {centralStatement: sg.centralStatement, activeUnknownNodeId: sg.activeUnknownNodeId, resolvedNodeIds: [...sg.resolvedNodeIds], nodeCount: sg.nodes.length, edgeCount: sg.edges.length};
|
|
const pc = {provider: "ollama", model: process.env.OLLAMA_MODEL || "configured-model"};
|
|
return {supportedEvidence: se, findings: fi, plausibleInterpretations: pi, unresolvedQuestions: includeUQ ? [...uq] : [], scenarioState: ss, providerConfig: pc};
|
|
} |