fix(confidence-engine): constrain understanding to supported evidence
This commit is contained in:
@@ -8,31 +8,52 @@ import {
|
||||
|
||||
// ── Fixtures ────────────────────────────────────────────────
|
||||
|
||||
const canonicalGraph = {
|
||||
centralStatement: "Revenue dropped 30% in Q2 due to supply chain disruption.",
|
||||
nodes: [
|
||||
{ id: "n1", proposition: "Q1 revenue was stable", description: "Baseline metric", status: "confirmed", confidence: 0.95 },
|
||||
{ id: "n2", proposition: "Supplier A failed deliveries in May", description: "Primary cause", status: "active", confidence: 0.85 },
|
||||
{ id: "n3", proposition: "Customer churn increased by 12%", description: "Secondary effect", status: "active", confidence: 0.7 },
|
||||
],
|
||||
edges: [
|
||||
{ from: "n2", to: "n3", type: "causal", context: "Supply failure led to customer dissatisfaction" },
|
||||
{ from: "n1", to: "n2", type: "temporal", context: "Preceding event in causal chain" },
|
||||
],
|
||||
/** A graph where only centralStatement is present — no known/supported nodes */
|
||||
const emptyGraph = {
|
||||
centralStatement: "Complaints increased by 35% while production increased by 40%.",
|
||||
};
|
||||
|
||||
const makeFinding = (overrides = {}) => ({
|
||||
id: `find-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
||||
proposition: "Supplier delays caused production halts",
|
||||
/** Full scenario fixture matching the fixed semantic regression case */
|
||||
const fullScenario = {
|
||||
centralStatement:
|
||||
"Complaints increased by 35% while production increased by 40%.",
|
||||
nodes: [
|
||||
{ id: "n1", label: "Complaint count increased by 35%", description: "Primary metric baseline", kind: "metric", status: "known", confidence: "high", value: 35, unit: "%" },
|
||||
{ id: "n2", label: "Production count increased by 40%", description: "Secondary metric baseline", kind: "metric", status: "supported", confidence: "medium", value: 40, unit: "%" },
|
||||
{ id: "n3", label: "Baseline period denominator volume", description: "Question about the baseline", kind: "unknown", status: "unknown", confidence: "low" },
|
||||
{ id: "n4", label: "Time period over which these percentage changes occurred", description: "Resolved question — user answered", kind: "question", status: "resolved", confidence: "high" },
|
||||
{ id: "n5", label: "Product mix may explain the complaint increase", description: "Provisional hypothesis for later investigation", kind: "hypothesis", status: "provisional", confidence: "low" },
|
||||
],
|
||||
edges: [
|
||||
{ fromNodeId: "n1", toNodeId: "n2", relationship: "compares_with", confidence: "high", description: "Correlation between complaints and production" },
|
||||
],
|
||||
activeUnknownNodeId: "n3",
|
||||
resolvedNodeIds: ["n4"],
|
||||
currentSummary: "Sentinel: currentSummary must not appear in synthesis",
|
||||
reasoningState: { stage: "analysis", status: "active", outcome: "pending" },
|
||||
};
|
||||
|
||||
/** Minimal eligible Finding — userDisposition null (working premise) */
|
||||
const workingFinding = () => ({
|
||||
id: "find-working",
|
||||
proposition: "The reported percentage changes correspond to the last financial quarter.",
|
||||
evaluation: "considered",
|
||||
userDisposition: null,
|
||||
sourceObservation: "obs-1",
|
||||
contributionId: "contrib-001",
|
||||
...overrides,
|
||||
});
|
||||
|
||||
// Fake provider factory for tests — always has generateReconstruction
|
||||
// If defaultResponse is passed, use it; otherwise default to success.
|
||||
/** Minimal eligible Finding — userDisposition agree */
|
||||
const agreedFinding = () => ({
|
||||
id: "find-agreed",
|
||||
proposition: "The percentage increase in complaints exceeds the threshold.",
|
||||
evaluation: "considered",
|
||||
userDisposition: "agree",
|
||||
sourceObservation: "obs-2",
|
||||
contributionId: "contrib-002",
|
||||
});
|
||||
|
||||
/** Fake provider factory for tests */
|
||||
const makeFakeProvider = (defaultResponse) => ({
|
||||
generateReconstruction: vi.fn(async () => {
|
||||
return typeof defaultResponse === "function"
|
||||
@@ -41,134 +62,267 @@ const makeFakeProvider = (defaultResponse) => ({
|
||||
}),
|
||||
});
|
||||
|
||||
// ── Eligibility tests ───────────────────────────────────────
|
||||
// ── Eligibility tests (unchanged — domain invariant) ────────
|
||||
|
||||
describe("filterEligibleFindings — eligibility contract", () => {
|
||||
it("includes null disposition → eligible", () => {
|
||||
const findings = [makeFinding({ userDisposition: null })];
|
||||
const result = filterEligibleFindings(findings);
|
||||
const result = filterEligibleFindings([workingFinding()]);
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].userDisposition).toBeNull();
|
||||
});
|
||||
|
||||
it("includes agree disposition → eligible", () => {
|
||||
const findings = [makeFinding({ userDisposition: "agree" })];
|
||||
const result = filterEligibleFindings(findings);
|
||||
const result = filterEligibleFindings([agreedFinding()]);
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].userDisposition).toBe("agree");
|
||||
});
|
||||
|
||||
it("excludes not_quite disposition → ineligible", () => {
|
||||
const findings = [makeFinding({ userDisposition: "not_quite" })];
|
||||
const result = filterEligibleFindings(findings);
|
||||
expect(result).toHaveLength(0);
|
||||
expect(filterEligibleFindings([{ userDisposition: "not_quite" }])).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("excludes not_relevant disposition → ineligible", () => {
|
||||
const findings = [makeFinding({ userDisposition: "not_relevant" })];
|
||||
const result = filterEligibleFindings(findings);
|
||||
expect(result).toHaveLength(0);
|
||||
expect(filterEligibleFindings([{ userDisposition: "not_relevant" }])).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("excludes rejected evaluation → excluded", () => {
|
||||
const findings = [makeFinding({ evaluation: "rejected" })];
|
||||
const result = filterEligibleFindings(findings);
|
||||
expect(result).toHaveLength(0);
|
||||
expect(filterEligibleFindings([{ evaluation: "rejected" }])).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("mixed dispositions — only eligible pass through", () => {
|
||||
const findings = [
|
||||
makeFinding({ userDisposition: null, id: "f1" }),
|
||||
makeFinding({ userDisposition: "agree", id: "f2" }),
|
||||
makeFinding({ userDisposition: "not_quite", id: "f3" }),
|
||||
makeFinding({ userDisposition: "not_relevant", id: "f4" }),
|
||||
makeFinding({ evaluation: "rejected", id: "f5" }),
|
||||
];
|
||||
const result = filterEligibleFindings(findings);
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result.map((f) => f.id)).toEqual(["f1", "f2"]);
|
||||
const results = filterEligibleFindings([
|
||||
{ id: "f1", userDisposition: null },
|
||||
{ id: "f2", userDisposition: "agree" },
|
||||
{ id: "f3", userDisposition: "not_quite" },
|
||||
{ id: "f4", userDisposition: "not_relevant" },
|
||||
{ evaluation: "rejected", id: "f5" },
|
||||
]);
|
||||
expect(results.map((f) => f.id)).toEqual(["f1", "f2"]);
|
||||
});
|
||||
|
||||
it("null input returns empty array", () => {
|
||||
it("null/undefined/empty input returns empty array", () => {
|
||||
expect(filterEligibleFindings(null)).toEqual([]);
|
||||
expect(filterEligibleFindings(undefined)).toEqual([]);
|
||||
expect(filterEligibleFindings([])).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Prompt content tests ────────────────────────────────────
|
||||
// ── Evidence projection — status filtering ──────────────────
|
||||
|
||||
describe("buildSynthesisPrompt — full graph input", () => {
|
||||
it("includes full canonical graph: nodes, edges, centralStatement (not just centralStatement)", () => {
|
||||
const prompt = buildSynthesisPrompt(canonicalGraph, []);
|
||||
expect(prompt).toContain("Canonical Situation Graph");
|
||||
expect(prompt).toContain("Revenue dropped 30%");
|
||||
describe("buildSynthesisPrompt — evidence-authority boundary", () => {
|
||||
// ── A. known retained ────────────────────────────────────
|
||||
|
||||
// Verify nodes are included with content beyond centralStatement
|
||||
expect(prompt).toContain('Node(n1)');
|
||||
expect(prompt).toContain("Q1 revenue was stable");
|
||||
expect(prompt).toContain('Node(n2)');
|
||||
expect(prompt).toContain("Supplier A failed deliveries in May");
|
||||
|
||||
// Verify edges are included
|
||||
expect(prompt).toContain("Edge(n2 → n3");
|
||||
expect(prompt).toContain("causal");
|
||||
expect(prompt).toContain("Edge(n1 → n2");
|
||||
expect(prompt).toContain("temporal");
|
||||
|
||||
// Verify centralStatement value is present (not just the key name)
|
||||
expect(prompt).toContain("supply chain disruption");
|
||||
it("known node appears in the synthesis prompt (label, kind, value, unit)", () => {
|
||||
const prompt = buildSynthesisPrompt(fullScenario, []);
|
||||
expect(prompt).toContain("Complaint count increased by 35%");
|
||||
expect(prompt).toContain("kind=metric");
|
||||
expect(prompt).toContain("[known]");
|
||||
});
|
||||
|
||||
it("includes eligible Finding propositions in prompt", () => {
|
||||
const findings = [makeFinding({ userDisposition: "agree" })];
|
||||
const prompt = buildSynthesisPrompt(canonicalGraph, findings);
|
||||
expect(prompt).toContain("Supplier delays caused production halts");
|
||||
// ── B. supported retained ────────────────────────────────
|
||||
|
||||
it("supported node appears in the synthesis prompt", () => {
|
||||
const prompt = buildSynthesisPrompt(fullScenario, []);
|
||||
expect(prompt).toContain("Production count increased by 40%");
|
||||
expect(prompt).toContain("[supported]");
|
||||
});
|
||||
|
||||
it("excludes non-eligible Finding propositions from prompt", () => {
|
||||
const ineligibleFindings = [makeFinding({ userDisposition: "not_quite" })];
|
||||
const eligibleFindings = filterEligibleFindings(ineligibleFindings);
|
||||
const prompt = buildSynthesisPrompt(canonicalGraph, eligibleFindings);
|
||||
expect(prompt).toContain("Eligible Findings");
|
||||
// The excluded proposition must not appear because eligible findings is empty
|
||||
expect(eligibleFindings).toHaveLength(0);
|
||||
// ── C. unknown excluded ──────────────────────────────────
|
||||
|
||||
it("unknown node content does NOT appear anywhere in synthesis", () => {
|
||||
const prompt = buildSynthesisPrompt(fullScenario, []);
|
||||
expect(prompt).not.toContain("Baseline period denominator volume");
|
||||
expect(prompt).not.toContain('status":"unknown"');
|
||||
expect(prompt).not.toContain("question about the baseline");
|
||||
});
|
||||
|
||||
it("includes provisional (null disposition) findings", () => {
|
||||
const findings = [makeFinding({ userDisposition: null })];
|
||||
const prompt = buildSynthesisPrompt(canonicalGraph, findings);
|
||||
expect(prompt).toContain("Provisional Findings");
|
||||
expect(prompt).toContain("Supplier delays caused production halts");
|
||||
// ── D. provisional excluded ──────────────────────────────
|
||||
|
||||
it("provisional hypothesis does NOT appear as CU evidence", () => {
|
||||
const prompt = buildSynthesisPrompt(fullScenario, []);
|
||||
expect(prompt).not.toContain("Product mix may explain");
|
||||
expect(prompt).not.toContain('status":"provisional"');
|
||||
});
|
||||
|
||||
it("includes confirmed (agree) findings", () => {
|
||||
const findings = [makeFinding({ userDisposition: "agree" })];
|
||||
const prompt = buildSynthesisPrompt(canonicalGraph, findings);
|
||||
// ── E. resolved-question text excluded ───────────────────
|
||||
|
||||
it("resolved node label/description does NOT appear as CU evidence", () => {
|
||||
const prompt = buildSynthesisPrompt(fullScenario, []);
|
||||
expect(prompt).not.toContain("Time period over which these percentage changes occurred");
|
||||
expect(prompt).not.toContain('status":"resolved"');
|
||||
expect(prompt).not.toContain("Resolved question");
|
||||
});
|
||||
|
||||
// ── F. learned Finding retained ──────────────────────────
|
||||
|
||||
it("eligible Finding appears in synthesis — proves we exclude resolved question while retaining the answer", () => {
|
||||
const prompt = buildSynthesisPrompt(fullScenario, [workingFinding()]);
|
||||
expect(prompt).toContain("The reported percentage changes correspond to the last financial quarter.");
|
||||
expect(prompt).not.toContain("Time period over which these percentage changes occurred");
|
||||
});
|
||||
|
||||
// ── G. control/reasoning content excluded ────────────────
|
||||
|
||||
it("control/reasoning sentinels are absent from synthesis prompt", () => {
|
||||
const prompt = buildSynthesisPrompt(fullScenario, []);
|
||||
expect(prompt).not.toContain("currentSummary");
|
||||
expect(prompt).not.toContain("Sentinel: currentSummary must not appear in synthesis");
|
||||
expect(prompt).not.toContain("reasoningState");
|
||||
expect(prompt).not.toContain("activeUnknownNodeId");
|
||||
expect(prompt).not.toContain('"n3"'); // active unknown node id sentinel
|
||||
expect(prompt).not.toContain("resolvedNodeIds");
|
||||
expect(prompt).not.toContain('"n4"'); // resolved node id sentinel
|
||||
});
|
||||
|
||||
// ── H. centralStatement retained as framing ─────────────
|
||||
|
||||
it("centralStatement remains available as framing context", () => {
|
||||
const prompt = buildSynthesisPrompt(fullScenario, []);
|
||||
expect(prompt).toContain("Complaints increased by 35% while production increased by 40%");
|
||||
});
|
||||
|
||||
// ── I. node.proposition dead read removed ────────────────
|
||||
|
||||
it("prompt does NOT depend on a proposition property on SituationGraph nodes", () => {
|
||||
const noPropositionGraph = {
|
||||
centralStatement: "Only framing.",
|
||||
nodes: [
|
||||
{ id: "a1", label: "Label only node", description: "No proposition field", kind: "observation", status: "known", confidence: "medium" },
|
||||
],
|
||||
};
|
||||
const prompt = buildSynthesisPrompt(noPropositionGraph, []);
|
||||
expect(prompt).toContain("Only framing.");
|
||||
expect(prompt).toContain("Label only node");
|
||||
// The projection must work correctly even when nodes lack a 'propertion' field entirely.
|
||||
});
|
||||
|
||||
// ── Structural correctness of the text-formatted evidence section ─
|
||||
|
||||
it("evidence section contains Known Facts and Supported Inferences headers", () => {
|
||||
const prompt = buildSynthesisPrompt(fullScenario, []);
|
||||
expect(prompt).toContain("Known Facts:");
|
||||
expect(prompt).toContain("Supported Inferences:");
|
||||
});
|
||||
|
||||
it("evidence section does not contain raw graph fields (edges, resolvedNodeIds, activeUnknownNodeId, reasoningState)", () => {
|
||||
const prompt = buildSynthesisPrompt(fullScenario, []);
|
||||
// These are all control/structural graph fields that must not appear in the evidence section
|
||||
expect(prompt).not.toContain("edges");
|
||||
expect(prompt).not.toContain("resolvedNodeIds");
|
||||
expect(prompt).not.toContain("activeUnknownNodeId");
|
||||
expect(prompt).not.toContain("reasoningState");
|
||||
});
|
||||
|
||||
it("node representation includes all canonical fields — kind, label, status, value, unit", () => {
|
||||
const prompt = buildSynthesisPrompt(fullScenario, []);
|
||||
// Each node line contains: kind=..., label="...", value=... [status]
|
||||
expect(prompt).toContain("kind=metric");
|
||||
expect(prompt).toContain('label="Complaint count increased by 35%"');
|
||||
expect(prompt).toContain("[known]");
|
||||
expect(prompt).toContain("[supported]");
|
||||
expect(prompt).toContain("value=35 (");
|
||||
expect(prompt).toContain("Primary metric baseline");
|
||||
});
|
||||
|
||||
it("provider sees structured evidence section header", () => {
|
||||
const prompt = buildSynthesisPrompt(fullScenario, []);
|
||||
expect(prompt).toContain("Provider-Active Evidence:");
|
||||
expect(prompt).not.toContain("Canonical Situation Graph");
|
||||
});
|
||||
|
||||
// ── Edge case: empty graph (no known/supported nodes) ────
|
||||
|
||||
it("handles graph with no evidence nodes gracefully", () => {
|
||||
const prompt = buildSynthesisPrompt(emptyGraph, []);
|
||||
expect(prompt).not.toContain("(no evidence)");
|
||||
expect(prompt).toContain("Complaints increased by 35% while production increased by 40%");
|
||||
});
|
||||
|
||||
// ── Edge case: only supported, no known ──────────────────
|
||||
|
||||
it("handles graph with only supported nodes", () => {
|
||||
const supportedOnly = {
|
||||
centralStatement: "Metric drift detected.",
|
||||
nodes: [
|
||||
{ id: "s1", label: "Drift exceeds threshold", description: "Supported inference", kind: "metric", status: "supported", confidence: "low" },
|
||||
],
|
||||
};
|
||||
const prompt = buildSynthesisPrompt(supportedOnly, []);
|
||||
expect(prompt).toContain("Drift exceeds threshold");
|
||||
});
|
||||
|
||||
// ── Finding integration tests ────────────────────────────
|
||||
|
||||
it("agreed Finding appears in synthesis with Correct Evidence label", () => {
|
||||
const prompt = buildSynthesisPrompt(fullScenario, [agreedFinding()]);
|
||||
expect(prompt).toContain("The percentage increase in complaints exceeds the threshold.");
|
||||
expect(prompt).toContain("Confirmed Evidence");
|
||||
});
|
||||
|
||||
it("handles zero eligible Findings — synthesis still proceeds from graph alone", () => {
|
||||
const prompt = buildSynthesisPrompt(canonicalGraph, []);
|
||||
expect(prompt).toContain("(none)");
|
||||
// Must still contain graph content
|
||||
expect(prompt).toContain("Canonical Situation Graph");
|
||||
it("working premise Finding appears with Working Premises label", () => {
|
||||
const prompt = buildSynthesisPrompt(fullScenario, [workingFinding()]);
|
||||
expect(prompt).toContain("The reported percentage changes correspond to the last financial quarter.");
|
||||
expect(prompt).toContain("Working Premises");
|
||||
});
|
||||
|
||||
it("prompt contains fresh-synthesis instructions (not append semantics)", () => {
|
||||
const prompt = buildSynthesisPrompt(canonicalGraph, []);
|
||||
expect(prompt).toContain("FRESH synthesis");
|
||||
expect(prompt).toContain("Do NOT treat any previous Current Understanding as input");
|
||||
expect(prompt).toContain("Do NOT append to prior summaries");
|
||||
// ── Immutability ─────────────────────────────────────────
|
||||
|
||||
it("graph structurally unchanged after synthesis prompt build", () => {
|
||||
const graphSnapshot = JSON.parse(JSON.stringify(fullScenario));
|
||||
buildSynthesisPrompt(fullScenario, []);
|
||||
expect(JSON.stringify(fullScenario)).toBe(JSON.stringify(graphSnapshot));
|
||||
});
|
||||
|
||||
it("prompt does NOT request graph mutations or Finding mutations", () => {
|
||||
const prompt = buildSynthesisPrompt(canonicalGraph, []);
|
||||
expect(prompt).not.toMatch(/change\s+selectedQuestion/i);
|
||||
it("findings structurally unchanged after synthesis prompt build", () => {
|
||||
const findings = [workingFinding()];
|
||||
const snapshot = JSON.parse(JSON.stringify(findings));
|
||||
buildSynthesisPrompt(fullScenario, findings);
|
||||
expect(JSON.stringify(findings)).toBe(JSON.stringify(snapshot));
|
||||
});
|
||||
|
||||
// ── Immutability during full seam ────────────────────────
|
||||
|
||||
it("graph and findings unchanged after synthesizeCurrentUnderstanding", async () => {
|
||||
const graphSnapshot = JSON.parse(JSON.stringify(fullScenario));
|
||||
const findings = [workingFinding()];
|
||||
const findingsSnapshot = JSON.parse(JSON.stringify(findings));
|
||||
|
||||
const fake = makeFakeProvider();
|
||||
await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: fullScenario, findings },
|
||||
{ provider: fake }
|
||||
);
|
||||
|
||||
expect(JSON.stringify(fullScenario)).toBe(JSON.stringify(graphSnapshot));
|
||||
expect(JSON.stringify(findings)).toBe(JSON.stringify(findingsSnapshot));
|
||||
});
|
||||
});
|
||||
|
||||
// ── Output validation tests ─────────────────────────────────
|
||||
// ── Prompt authority rule verification ─────────────────────
|
||||
|
||||
describe("buildSynthesisPrompt — prompt semantics", () => {
|
||||
it("prompt contains evidence-authority boundary rule (not open questions)", () => {
|
||||
const prompt = buildSynthesisPrompt(fullScenario, []);
|
||||
expect(prompt).toContain("Current Understanding describes only established or supported understanding");
|
||||
expect(prompt).toContain("Do not introduce or describe open questions, unresolved uncertainties, assumptions, provisional hypotheses");
|
||||
});
|
||||
|
||||
it("prompt clarifies centralStatement is framing context only", () => {
|
||||
const prompt = buildSynthesisPrompt(fullScenario, []);
|
||||
expect(prompt).toContain("Framing");
|
||||
// Must explicitly say the centralStatement is not independent evidence
|
||||
expect(prompt).toContain("framing context, not independent evidence");
|
||||
});
|
||||
|
||||
it("prompt does NOT include edges in the evidence section", () => {
|
||||
const prompt = buildSynthesisPrompt(fullScenario, []);
|
||||
// Edges should not be mentioned in the structured evidence projection
|
||||
expect(prompt).not.toContain("Edge");
|
||||
expect(prompt).not.toMatch(/relationship:.*compares_with/);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Output validation tests (unchanged) ────────────────────
|
||||
|
||||
describe("validateSynthesisResponse", () => {
|
||||
it("accepts valid narrative JSON object", () => {
|
||||
@@ -199,153 +353,120 @@ describe("validateSynthesisResponse", () => {
|
||||
});
|
||||
|
||||
it("rejects null input", () => {
|
||||
const result = validateSynthesisResponse(null);
|
||||
expect(result.valid).toBe(false);
|
||||
expect(validateSynthesisResponse(null).valid).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects undefined input", () => {
|
||||
const result = validateSynthesisResponse(undefined);
|
||||
expect(result.valid).toBe(false);
|
||||
expect(validateSynthesisResponse(undefined).valid).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Domain function tests ───────────────────────────────────
|
||||
// ── Domain function tests (full seam) ──────────────────────
|
||||
|
||||
describe("synthesizeCurrentUnderstanding — full seam", () => {
|
||||
it("null disposition finding → included in synthesis", async () => {
|
||||
const fake = makeFakeProvider();
|
||||
const result = await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: canonicalGraph, findings: [makeFinding({ userDisposition: null })] },
|
||||
{ provider: fake }
|
||||
);
|
||||
expect(result.currentUnderstanding).toBe("Synthesized output");
|
||||
expect(fake.generateReconstruction).toHaveBeenCalledTimes(1);
|
||||
const prompt = fake.generateReconstruction.mock.calls[0][0];
|
||||
expect(prompt).toContain("Supplier delays caused production halts");
|
||||
});
|
||||
// ── Evidence inclusion/exclusion at seam level ───────────
|
||||
|
||||
it("agree disposition finding → included in synthesis", async () => {
|
||||
const fake = makeFakeProvider();
|
||||
const result = await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: canonicalGraph, findings: [makeFinding({ userDisposition: "agree" })] },
|
||||
{ provider: fake }
|
||||
);
|
||||
expect(result.currentUnderstanding).toBe("Synthesized output");
|
||||
});
|
||||
|
||||
it("not_quite disposition finding → excluded from synthesis", async () => {
|
||||
const fake = makeFakeProvider();
|
||||
const result = await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: canonicalGraph, findings: [makeFinding({ userDisposition: "not_quite" })] },
|
||||
{ provider: fake }
|
||||
);
|
||||
expect(result.currentUnderstanding).toBe("Synthesized output");
|
||||
const prompt = fake.generateReconstruction.mock.calls[0][0];
|
||||
// The not_quite proposition must NOT appear because it was excluded by eligibility
|
||||
expect(prompt).toContain("(none)");
|
||||
expect(prompt).not.toContain("Supplier delays caused production halts");
|
||||
});
|
||||
|
||||
it("not_relevant disposition finding → excluded from synthesis", async () => {
|
||||
it("known node content flows to provider via synthesis prompt", async () => {
|
||||
const fake = makeFakeProvider();
|
||||
await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: canonicalGraph, findings: [makeFinding({ userDisposition: "not_relevant" })] },
|
||||
{ situationGraph: fullScenario, findings: [] },
|
||||
{ provider: fake }
|
||||
);
|
||||
const prompt = fake.generateReconstruction.mock.calls[0][0];
|
||||
expect(prompt).toContain("(none)");
|
||||
expect(prompt).toContain("Complaint count increased by 35%");
|
||||
expect(prompt).toContain("[known]");
|
||||
});
|
||||
|
||||
it("rejected evaluation → excluded from synthesis", async () => {
|
||||
it("supported node content flows to provider via synthesis prompt", async () => {
|
||||
const fake = makeFakeProvider();
|
||||
await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: canonicalGraph, findings: [makeFinding({ evaluation: "rejected" })] },
|
||||
{ situationGraph: fullScenario, findings: [] },
|
||||
{ provider: fake }
|
||||
);
|
||||
const prompt = fake.generateReconstruction.mock.calls[0][0];
|
||||
expect(prompt).toContain("(none)");
|
||||
expect(prompt).toContain("Production count increased by 40%");
|
||||
expect(prompt).toContain("[supported]");
|
||||
});
|
||||
|
||||
it("zero eligible Findings → synthesis succeeds from graph alone", async () => {
|
||||
it("unknown nodes do NOT appear in provider-visible synthesis", async () => {
|
||||
const fake = makeFakeProvider();
|
||||
const result = await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: canonicalGraph, findings: [] },
|
||||
await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: fullScenario, findings: [] },
|
||||
{ provider: fake }
|
||||
);
|
||||
expect(result.currentUnderstanding).toBe("Synthesized output");
|
||||
const prompt = fake.generateReconstruction.mock.calls[0][0];
|
||||
// Must still contain graph content
|
||||
expect(prompt).toContain("Canonical Situation Graph");
|
||||
expect(prompt).toContain("Node(n1)");
|
||||
expect(prompt).not.toContain("Baseline period denominator volume");
|
||||
});
|
||||
|
||||
it("fake provider returns valid narrative → { currentUnderstanding }", async () => {
|
||||
it("provisional hypothesis does NOT appear in synthesis prompt", async () => {
|
||||
const fake = makeFakeProvider();
|
||||
const result = await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: canonicalGraph, findings: [makeFinding({ userDisposition: "agree" })] },
|
||||
await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: fullScenario, findings: [] },
|
||||
{ provider: fake }
|
||||
);
|
||||
expect(result).toEqual({ currentUnderstanding: "Synthesized output" });
|
||||
const prompt = fake.generateReconstruction.mock.calls[0][0];
|
||||
expect(prompt).not.toContain("Product mix may explain");
|
||||
});
|
||||
|
||||
it("fake provider returns empty narrative → rejected", async () => {
|
||||
const fake = makeFakeProvider(async () => JSON.stringify({ currentUnderstanding: "" }));
|
||||
await expect(
|
||||
synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: canonicalGraph, findings: [] },
|
||||
{ provider: fake }
|
||||
)
|
||||
).rejects.toThrow(/Synthesis validation failed/);
|
||||
it("resolved-question text does NOT appear in synthesis prompt", async () => {
|
||||
const fake = makeFakeProvider();
|
||||
await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: fullScenario, findings: [] },
|
||||
{ provider: fake }
|
||||
);
|
||||
const prompt = fake.generateReconstruction.mock.calls[0][0];
|
||||
expect(prompt).not.toContain("Time period over which these percentage changes occurred");
|
||||
});
|
||||
|
||||
it("fake provider returns malformed JSON → rejected", async () => {
|
||||
const fake = makeFakeProvider(async () => "{ not valid json");
|
||||
await expect(
|
||||
synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: canonicalGraph, findings: [] },
|
||||
{ provider: fake }
|
||||
)
|
||||
).rejects.toThrow(/Synthesis validation failed/);
|
||||
it("eligible Finding appears in synthesis", async () => {
|
||||
const fake = makeFakeProvider();
|
||||
await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: fullScenario, findings: [workingFinding()] },
|
||||
{ provider: fake }
|
||||
);
|
||||
const prompt = fake.generateReconstruction.mock.calls[0][0];
|
||||
expect(prompt).toContain("The reported percentage changes correspond to the last financial quarter.");
|
||||
});
|
||||
|
||||
it("fake provider returns object without currentUnderstanding → rejected", async () => {
|
||||
const fake = makeFakeProvider(async () => JSON.stringify({ wrongField: "value" }));
|
||||
await expect(
|
||||
synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: canonicalGraph, findings: [] },
|
||||
{ provider: fake }
|
||||
)
|
||||
).rejects.toThrow(/Synthesis validation failed/);
|
||||
it("control/reasoning sentinels excluded from synthesis", async () => {
|
||||
const fake = makeFakeProvider();
|
||||
await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: fullScenario, findings: [] },
|
||||
{ provider: fake }
|
||||
);
|
||||
const prompt = fake.generateReconstruction.mock.calls[0][0];
|
||||
expect(prompt).not.toContain("currentSummary");
|
||||
expect(prompt).not.toContain("reasoningState");
|
||||
});
|
||||
|
||||
// ── Immutability tests ────────────────────────────────────
|
||||
// ── Immutability ─────────────────────────────────────────
|
||||
|
||||
it("graph structurally unchanged after synthesis", async () => {
|
||||
const graphSnapshot = JSON.parse(JSON.stringify(canonicalGraph));
|
||||
const graphSnapshot = JSON.parse(JSON.stringify(fullScenario));
|
||||
const fake = makeFakeProvider();
|
||||
|
||||
await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: canonicalGraph, findings: [] },
|
||||
{ situationGraph: fullScenario, findings: [] },
|
||||
{ provider: fake }
|
||||
);
|
||||
|
||||
expect(JSON.stringify(canonicalGraph)).toBe(JSON.stringify(graphSnapshot));
|
||||
expect(JSON.stringify(fullScenario)).toBe(JSON.stringify(graphSnapshot));
|
||||
});
|
||||
|
||||
it("findings structurally unchanged after synthesis", async () => {
|
||||
const findings = [makeFinding({ userDisposition: "agree" })];
|
||||
const findings = [workingFinding()];
|
||||
const findingsSnapshot = JSON.parse(JSON.stringify(findings));
|
||||
const fake = makeFakeProvider();
|
||||
|
||||
await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: canonicalGraph, findings },
|
||||
{ situationGraph: fullScenario, findings },
|
||||
{ provider: fake }
|
||||
);
|
||||
|
||||
expect(JSON.stringify(findings)).toBe(JSON.stringify(findingsSnapshot));
|
||||
});
|
||||
|
||||
// ── Input validation tests ────────────────────────────────
|
||||
// ── Input validation (unchanged) ────────────────────────
|
||||
|
||||
it("missing situationGraph → throws 400", async () => {
|
||||
await expect(
|
||||
@@ -361,7 +482,7 @@ describe("synthesizeCurrentUnderstanding — full seam", () => {
|
||||
|
||||
it("findings as non-array → throws 400", async () => {
|
||||
await expect(
|
||||
synthesizeCurrentUnderstanding({ situationGraph: canonicalGraph, findings: "string" }, { provider: makeFakeProvider() })
|
||||
synthesizeCurrentUnderstanding({ situationGraph: fullScenario, findings: "string" }, { provider: makeFakeProvider() })
|
||||
).rejects.toThrow(/findings must be an array/);
|
||||
});
|
||||
|
||||
@@ -369,47 +490,36 @@ describe("synthesizeCurrentUnderstanding — full seam", () => {
|
||||
const fake = makeFakeProvider(async () => { throw new Error("provider down"); });
|
||||
await expect(
|
||||
synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: canonicalGraph, findings: [] },
|
||||
{ situationGraph: fullScenario, findings: [] },
|
||||
{ provider: fake }
|
||||
)
|
||||
).rejects.toThrow(/Synthesis provider call failed|provider down/);
|
||||
});
|
||||
|
||||
it("no provider provided — falls through to getProvider() which needs env vars", async () => {
|
||||
it("no provider provided — falls through to getProvider()", async () => {
|
||||
await expect(
|
||||
synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: canonicalGraph, findings: [] },
|
||||
{ situationGraph: fullScenario, findings: [] },
|
||||
{}
|
||||
)
|
||||
).rejects.toThrow(/OLLAMA_BASE_URL/);
|
||||
});
|
||||
|
||||
// ── Configured model resolution (unchanged) ─────────────
|
||||
|
||||
// ── Configured model resolution ──────────────────────────
|
||||
|
||||
it("default synthesis path resolves configured modelName — provider receives non-null", async () => {
|
||||
it("default synthesis path resolves configured modelName", async () => {
|
||||
const fake = makeFakeProvider();
|
||||
|
||||
// Stub the configured model so test does not depend on dev-machine .env.local
|
||||
const savedModel = process.env.OLLAMA_MODEL;
|
||||
process.env.OLLAMA_MODEL = "configured-model-v0.50";
|
||||
|
||||
try {
|
||||
const result = await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: canonicalGraph, findings: [] },
|
||||
await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: fullScenario, findings: [] },
|
||||
{ provider: fake }
|
||||
);
|
||||
|
||||
expect(result.currentUnderstanding).toBe("Synthesized output");
|
||||
|
||||
// KEY ASSERTION: configured model must flow to provider
|
||||
const receivedModel = fake.generateReconstruction.mock.calls[0][1];
|
||||
expect(receivedModel).toBeDefined();
|
||||
expect(receivedModel).not.toBeNull();
|
||||
expect(typeof receivedModel).toBe("string");
|
||||
expect(receivedModel.length).toBeGreaterThan(0);
|
||||
} finally {
|
||||
// Restore original env value (may be undefined)
|
||||
if (savedModel == null) {
|
||||
delete process.env.OLLAMA_MODEL;
|
||||
} else {
|
||||
@@ -418,37 +528,96 @@ describe("synthesizeCurrentUnderstanding — full seam", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("explicit modelName dependency overrides default — provider receives injected model", async () => {
|
||||
it("explicit modelName dependency overrides default", async () => {
|
||||
const fake = makeFakeProvider();
|
||||
await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: canonicalGraph, findings: [] },
|
||||
{ situationGraph: fullScenario, findings: [] },
|
||||
{ provider: fake, modelName: "test-model-v0.50" }
|
||||
);
|
||||
|
||||
expect(fake.generateReconstruction.mock.calls[0][1]).toBe("test-model-v0.50");
|
||||
});
|
||||
|
||||
// ── Provider sees full graph, not just centralStatement ───
|
||||
// ── Provider sees evidence projection, not raw graph ─────
|
||||
|
||||
it("provider receives full canonical graph content (nodes + edges + centralStatement)", async () => {
|
||||
it("provider receives structured evidence projection (not full graph dump)", async () => {
|
||||
const fake = makeFakeProvider();
|
||||
await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: canonicalGraph, findings: [] },
|
||||
{ situationGraph: fullScenario, findings: [] },
|
||||
{ provider: fake }
|
||||
);
|
||||
|
||||
const prompt = fake.generateReconstruction.mock.calls[0][0];
|
||||
// Must contain node content (not just centralStatement)
|
||||
expect(prompt).toContain("Node(n1)");
|
||||
expect(prompt).toContain("Q1 revenue was stable");
|
||||
expect(prompt).toContain("Node(n2)");
|
||||
expect(prompt).toContain("Supplier A failed deliveries in May");
|
||||
// Must contain edge content
|
||||
expect(prompt).toContain("Edge(n2 → n3");
|
||||
expect(prompt).toContain("causal");
|
||||
// Must contain centralStatement value
|
||||
expect(prompt).toContain("Revenue dropped 30%");
|
||||
// Must instruct about fresh synthesis (not previous CU)
|
||||
expect(prompt).toContain("Do NOT treat any previous Current Understanding as input");
|
||||
// Evidence section must be structured, not a raw JSON dump of the whole graph
|
||||
expect(prompt).toContain("Provider-Active Evidence");
|
||||
expect(prompt).not.toContain("Canonical Situation Graph");
|
||||
});
|
||||
|
||||
it("provider receives centralStatement as framing context", async () => {
|
||||
const fake = makeFakeProvider();
|
||||
await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: fullScenario, findings: [] },
|
||||
{ provider: fake }
|
||||
);
|
||||
const prompt = fake.generateReconstruction.mock.calls[0][0];
|
||||
expect(prompt).toContain("Complaints increased by 35%");
|
||||
expect(prompt).toContain("Framing");
|
||||
});
|
||||
|
||||
it("provider receives centralStatement framing for no-evidence graph", async () => {
|
||||
const fake = makeFakeProvider();
|
||||
await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: emptyGraph, findings: [] },
|
||||
{ provider: fake }
|
||||
);
|
||||
const prompt = fake.generateReconstruction.mock.calls[0][0];
|
||||
// Even with no evidence nodes, centralStatement framing flows through
|
||||
expect(prompt).toContain("Central Statement");
|
||||
});
|
||||
|
||||
it("null/undefined findings → synthesis proceeds with empty evidence", async () => {
|
||||
const fake = makeFakeProvider();
|
||||
await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: fullScenario, findings: null },
|
||||
{ provider: fake }
|
||||
);
|
||||
expect(fake.generateReconstruction).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("fake provider returns valid narrative → { currentUnderstanding }", async () => {
|
||||
const fake = makeFakeProvider();
|
||||
const result = await synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: fullScenario, findings: [workingFinding()] },
|
||||
{ provider: fake }
|
||||
);
|
||||
expect(result).toEqual({ currentUnderstanding: "Synthesized output" });
|
||||
});
|
||||
|
||||
it("fake provider returns empty narrative → rejected", async () => {
|
||||
const fake = makeFakeProvider(async () => JSON.stringify({ currentUnderstanding: "" }));
|
||||
await expect(
|
||||
synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: fullScenario, findings: [] },
|
||||
{ provider: fake }
|
||||
)
|
||||
).rejects.toThrow(/Synthesis validation failed/);
|
||||
});
|
||||
|
||||
it("fake provider returns malformed JSON → rejected", async () => {
|
||||
const fake = makeFakeProvider(async () => "{ not valid json");
|
||||
await expect(
|
||||
synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: fullScenario, findings: [] },
|
||||
{ provider: fake }
|
||||
)
|
||||
).rejects.toThrow(/Synthesis validation failed/);
|
||||
});
|
||||
|
||||
it("fake provider returns object without currentUnderstanding → rejected", async () => {
|
||||
const fake = makeFakeProvider(async () => JSON.stringify({ wrongField: "value" }));
|
||||
await expect(
|
||||
synthesizeCurrentUnderstanding(
|
||||
{ situationGraph: fullScenario, findings: [] },
|
||||
{ provider: fake }
|
||||
)
|
||||
).rejects.toThrow(/Synthesis validation failed/);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user