robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(confidence-engine): constrain understanding to supported evidence

Browse Source
This commit is contained in:
robbond
2026-09-01 18:17:26 +01:00
parent 161527f66c
commit b647236d44
2 changed files with 472 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/graph/current-understanding-synthesis.js
+84 -36
View File
@@ -52,31 +52,66 @@ export function filterEligibleFindings(findings) {
* The prompt instructs the model to produce one coherent Current Understanding narrative
* from the provided inputs, without append semantics.
*/
export function buildSynthesisPrompt(situationGraph, findings) {
// Build structured graph representation for the prompt
const graphInfo = {
centralStatement: situationGraph.centralStatement ?? "",
nodes: (situationGraph.nodes ?? []).map((n) => ({
id: n.id,
proposition: n.proposition ?? "",
description: n.description ?? "",
status: n.status ?? null,
confidence: n.confidence ?? null,
})),
edges: (situationGraph.edges ?? []).map((e) => ({
from: e.from ?? null,
to: e.to ?? null,
type: e.type ?? "",
context: e.context ?? "",
})),
};
// Build a structured representation of eligible Findings for the prompt
const KNOWN_SUPPORTED_STATUSES = new Set(["known", "supported"]);
function safeDesc(value) {
return (value && typeof value === "string") ? value : null;
}
/**
* Evidence-authority projection for CU synthesis.
*
* Includes only:
* - centralStatement (framing context — not independent evidence)
* - known nodes (provider-active graph evidence)
* - supported nodes (provider-active graph evidence)
* - eligible Findings (from the findings parameter)
*
* Explicitly excludes from provider-active synthesis input:
* provisional nodes, unknown nodes, resolved nodes, edges,
* activeUnknownNodeId, resolvedNodeIds, currentSummary,
* reasoningState, confidence/control metadata, dependency/control fields.
*/
function buildGraphEvidenceProjection(graphInfo) {
const knownNodes = (graphInfo.nodes ?? [])
.filter((n) => KNOWN_SUPPORTED_STATUSES.has(n.status))
.map((n) => ({
kind: n.kind ?? null,
label: safeDesc(n.label),
description: safeDesc(n.description),
value: n.value ?? null,
unit: n.unit ?? null,
status: n.status ?? null,
}));
const supportedNodes = (graphInfo.nodes ?? [])
.filter((n) => KNOWN_SUPPORTED_STATUSES.has(n.status))
.filter((n) => n.status !== "known")
.map((n) => ({
kind: n.kind ?? null,
label: safeDesc(n.label),
description: safeDesc(n.description),
value: n.value ?? null,
unit: n.unit ?? null,
status: n.status ?? null,
}));
const centralStatement = safeDesc(graphInfo.centralStatement) || "";
return { centralStatement, knownNodes, supportedNodes };
}
export function buildSynthesisPrompt(situationGraph, findings) {
// Structured evidence projection for the model
const evidence = buildGraphEvidenceProjection(situationGraph);
// Eligible Finding sections (preserve existing agree vs null distinction)
const findingsSections = [];
if (findings.length > 0) {
const agreed = findings.filter((f) => f.userDisposition === "agree");
const provisional = findings.filter(
const working = findings.filter(
(f) => f.userDisposition === null
);
@@ -90,10 +125,10 @@ export function buildSynthesisPrompt(situationGraph, findings) {
});
}
if (provisional.length > 0) {
if (working.length > 0) {
findingsSections.push({
label: "Provisional Findings (working interpretation)",
items: provisional.map((f) => ({
label: "Working Premises",
items: working.map((f) => ({
proposition: f.proposition,
id: f.id ?? null,
})),
@@ -101,14 +136,26 @@ export function buildSynthesisPrompt(situationGraph, findings) {
}
}
// Human-readable node and edge representations for the prompt
const nodesSection = graphInfo.nodes.length > 0 ? `\nNodes:\n${graphInfo.nodes.map((n) => ` Node(${n.id}): ${n.proposition}${n.description ? ` — ${n.description}` : ""}${n.status ? ` [${n.status}]` : ""}`).join("\n")}` : "";
const edgesSection = graphInfo.edges.length > 0 ? `\nEdges:\n${graphInfo.edges.map((e) => ` Edge(${e.from} → ${e.to}, type=${e.type}): ${e.context || "(no context)"}`).join("\n")}` : "";
// Human-readable node display for the prompt
function formatNodeSection(title, nodes) {
if (!nodes || nodes.length === 0) return "";
const items = nodes.map(
(n) => ` ${title}: kind=${n.kind}, label="${n.label}", value=${n.value ? n.value + (n.unit ? " (" + n.unit + ")" : "") : null} — ${n.description ?? "(no description)"} [${n.status}]`
);
return "\n" + items.join("\n");
}
const knownSection = formatNodeSection("Known", evidence.knownNodes);
const supportedItem = formatNodeSection("Supported", evidence.supportedNodes);
const prompt = `You are producing a Current Understanding narrative from investigation evidence.
Canonical Situation Graph:
${JSON.stringify(graphInfo, null, 2)}${nodesSection}${edgesSection}
Situation Framing:
${evidence.centralStatement ? " Central Statement: " + evidence.centralStatement : "(none)"}
Provider-Active Evidence:
Known Facts:${knownSection}
Supported Inferences:${supportedItem}
Eligible Findings:
${findingsSections.length > 0
@@ -116,14 +163,15 @@ ${findingsSections.length > 0
: "(none)"}
Rules for this synthesis:
1. Produce exactly ONE coherent narrative paragraph (or short multi-sentence paragraph) that represents the Current Understanding of the situation.
2. Synthesize all provided evidence into a unified understanding — do not list or append findings. The result should read as a natural summary, not a bullet list.
3. This is a FRESH synthesis from the complete set of inputs above. Do NOT treat any previous Current Understanding as input or authority. Do NOT append to prior summaries.
4. Use only information present in the Situation context and Eligible Findings above.
5. If no eligible Findings are provided, synthesize from the Situation context alone.
6. Return ONLY a JSON object with this exact structure:
1. Current Understanding describes only established or supported understanding from the evidence supplied here.
2. Do not introduce or describe open questions, unresolved uncertainties, assumptions, provisional hypotheses, speculative explanations, or future investigation needs.
3. Produce exactly ONE coherent narrative paragraph (or short multi-sentence paragraph) that represents the Current Understanding of the situation.
4. Synthesize all provided evidence into a unified understanding — do not list or append findings. The result should read as a natural summary, not a bullet list.
5. This is a FRESH synthesis from the complete set of inputs above. Do NOT treat any previous Current Understanding as input or authority. Do NOT append to prior summaries.
6. Use only information present in the evidence above. The centralStatement is framing context, not independent evidence.
7. Return ONLY a JSON object with this exact structure:
{"currentUnderstanding": "your narrative here"}
7. The currentUnderstanding value must be a non-empty string.
8. The currentUnderstanding value must be a non-empty string.
Return ONLY the JSON object. No markdown, no explanation, no preamble.`;
@@ -204,7 +252,7 @@ export async function synthesizeCurrentUnderstanding(
// 2. Eligibility normalization (domain seam responsibility)
const eligibleFindings = filterEligibleFindings(allFindings);
// 3. Build synthesis prompt (uses full canonical graph, not just centralStatement)
// 3. Build synthesis prompt (uses evidence-authority projection, not raw graph)
const prompt = buildSynthesisPrompt(situationGraph, eligibleFindings);
// 4. Resolve provider — DI fallback to configured default
tests/graph/current-understanding-synthesis.test.js
+388 -219
View File
@@ -8,31 +8,52 @@ import {
// ── Fixtures ────────────────────────────────────────────────
const canonicalGraph = {
centralStatement: "Revenue dropped 30% in Q2 due to supply chain disruption.",
nodes: [
{ id: "n1", proposition: "Q1 revenue was stable", description: "Baseline metric", status: "confirmed", confidence: 0.95 },
{ id: "n2", proposition: "Supplier A failed deliveries in May", description: "Primary cause", status: "active", confidence: 0.85 },
{ id: "n3", proposition: "Customer churn increased by 12%", description: "Secondary effect", status: "active", confidence: 0.7 },
],
edges: [
{ from: "n2", to: "n3", type: "causal", context: "Supply failure led to customer dissatisfaction" },
{ from: "n1", to: "n2", type: "temporal", context: "Preceding event in causal chain" },
],
/** A graph where only centralStatement is present — no known/supported nodes */
const emptyGraph = {
centralStatement: "Complaints increased by 35% while production increased by 40%.",
};
const makeFinding = (overrides = {}) => ({
id: `find-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
proposition: "Supplier delays caused production halts",
/** Full scenario fixture matching the fixed semantic regression case */
const fullScenario = {
centralStatement:
"Complaints increased by 35% while production increased by 40%.",
nodes: [
{ id: "n1", label: "Complaint count increased by 35%", description: "Primary metric baseline", kind: "metric", status: "known", confidence: "high", value: 35, unit: "%" },
{ id: "n2", label: "Production count increased by 40%", description: "Secondary metric baseline", kind: "metric", status: "supported", confidence: "medium", value: 40, unit: "%" },
{ id: "n3", label: "Baseline period denominator volume", description: "Question about the baseline", kind: "unknown", status: "unknown", confidence: "low" },
{ id: "n4", label: "Time period over which these percentage changes occurred", description: "Resolved question — user answered", kind: "question", status: "resolved", confidence: "high" },
{ id: "n5", label: "Product mix may explain the complaint increase", description: "Provisional hypothesis for later investigation", kind: "hypothesis", status: "provisional", confidence: "low" },
],
edges: [
{ fromNodeId: "n1", toNodeId: "n2", relationship: "compares_with", confidence: "high", description: "Correlation between complaints and production" },
],
activeUnknownNodeId: "n3",
resolvedNodeIds: ["n4"],
currentSummary: "Sentinel: currentSummary must not appear in synthesis",
reasoningState: { stage: "analysis", status: "active", outcome: "pending" },
};
/** Minimal eligible Finding — userDisposition null (working premise) */
const workingFinding = () => ({
id: "find-working",
proposition: "The reported percentage changes correspond to the last financial quarter.",
evaluation: "considered",
userDisposition: null,
sourceObservation: "obs-1",
contributionId: "contrib-001",
...overrides,
});
// Fake provider factory for tests — always has generateReconstruction
// If defaultResponse is passed, use it; otherwise default to success.
/** Minimal eligible Finding — userDisposition agree */
const agreedFinding = () => ({
id: "find-agreed",
proposition: "The percentage increase in complaints exceeds the threshold.",
evaluation: "considered",
userDisposition: "agree",
sourceObservation: "obs-2",
contributionId: "contrib-002",
});
/** Fake provider factory for tests */
const makeFakeProvider = (defaultResponse) => ({
generateReconstruction: vi.fn(async () => {
return typeof defaultResponse === "function"
@@ -41,134 +62,267 @@ const makeFakeProvider = (defaultResponse) => ({
}),
});
// ── Eligibility tests ───────────────────────────────────────
// ── Eligibility tests (unchanged — domain invariant) ────────
describe("filterEligibleFindings — eligibility contract", () => {
it("includes null disposition → eligible", () => {
const findings = [makeFinding({ userDisposition: null })];
const result = filterEligibleFindings(findings);
const result = filterEligibleFindings([workingFinding()]);
expect(result).toHaveLength(1);
expect(result[0].userDisposition).toBeNull();
});
it("includes agree disposition → eligible", () => {
const findings = [makeFinding({ userDisposition: "agree" })];
const result = filterEligibleFindings(findings);
const result = filterEligibleFindings([agreedFinding()]);
expect(result).toHaveLength(1);
expect(result[0].userDisposition).toBe("agree");
});
it("excludes not_quite disposition → ineligible", () => {
const findings = [makeFinding({ userDisposition: "not_quite" })];
const result = filterEligibleFindings(findings);
expect(result).toHaveLength(0);
expect(filterEligibleFindings([{ userDisposition: "not_quite" }])).toHaveLength(0);
});
it("excludes not_relevant disposition → ineligible", () => {
const findings = [makeFinding({ userDisposition: "not_relevant" })];
const result = filterEligibleFindings(findings);
expect(result).toHaveLength(0);
expect(filterEligibleFindings([{ userDisposition: "not_relevant" }])).toHaveLength(0);
});
it("excludes rejected evaluation → excluded", () => {
const findings = [makeFinding({ evaluation: "rejected" })];
const result = filterEligibleFindings(findings);
expect(result).toHaveLength(0);
expect(filterEligibleFindings([{ evaluation: "rejected" }])).toHaveLength(0);
});
it("mixed dispositions — only eligible pass through", () => {
const findings = [
makeFinding({ userDisposition: null, id: "f1" }),
makeFinding({ userDisposition: "agree", id: "f2" }),
makeFinding({ userDisposition: "not_quite", id: "f3" }),
makeFinding({ userDisposition: "not_relevant", id: "f4" }),
makeFinding({ evaluation: "rejected", id: "f5" }),
];
const result = filterEligibleFindings(findings);
expect(result).toHaveLength(2);
expect(result.map((f) => f.id)).toEqual(["f1", "f2"]);
const results = filterEligibleFindings([
{ id: "f1", userDisposition: null },
{ id: "f2", userDisposition: "agree" },
{ id: "f3", userDisposition: "not_quite" },
{ id: "f4", userDisposition: "not_relevant" },
{ evaluation: "rejected", id: "f5" },
]);
expect(results.map((f) => f.id)).toEqual(["f1", "f2"]);
});
it("null input returns empty array", () => {
it("null/undefined/empty input returns empty array", () => {
expect(filterEligibleFindings(null)).toEqual([]);
expect(filterEligibleFindings(undefined)).toEqual([]);
expect(filterEligibleFindings([])).toEqual([]);
});
});
// ── Prompt content tests ────────────────────────────────────
// ── Evidence projection — status filtering ──────────────────
describe("buildSynthesisPrompt — full graph input", () => {
it("includes full canonical graph: nodes, edges, centralStatement (not just centralStatement)", () => {
const prompt = buildSynthesisPrompt(canonicalGraph, []);
expect(prompt).toContain("Canonical Situation Graph");
expect(prompt).toContain("Revenue dropped 30%");
describe("buildSynthesisPrompt — evidence-authority boundary", () => {
// ── A. known retained ────────────────────────────────────
// Verify nodes are included with content beyond centralStatement
expect(prompt).toContain('Node(n1)');
expect(prompt).toContain("Q1 revenue was stable");
expect(prompt).toContain('Node(n2)');
expect(prompt).toContain("Supplier A failed deliveries in May");
// Verify edges are included
expect(prompt).toContain("Edge(n2 → n3");
expect(prompt).toContain("causal");
expect(prompt).toContain("Edge(n1 → n2");
expect(prompt).toContain("temporal");
// Verify centralStatement value is present (not just the key name)
expect(prompt).toContain("supply chain disruption");
it("known node appears in the synthesis prompt (label, kind, value, unit)", () => {
const prompt = buildSynthesisPrompt(fullScenario, []);
expect(prompt).toContain("Complaint count increased by 35%");
expect(prompt).toContain("kind=metric");
expect(prompt).toContain("[known]");
});
it("includes eligible Finding propositions in prompt", () => {
const findings = [makeFinding({ userDisposition: "agree" })];
const prompt = buildSynthesisPrompt(canonicalGraph, findings);
expect(prompt).toContain("Supplier delays caused production halts");
// ── B. supported retained ────────────────────────────────
it("supported node appears in the synthesis prompt", () => {
const prompt = buildSynthesisPrompt(fullScenario, []);
expect(prompt).toContain("Production count increased by 40%");
expect(prompt).toContain("[supported]");
});
it("excludes non-eligible Finding propositions from prompt", () => {
const ineligibleFindings = [makeFinding({ userDisposition: "not_quite" })];
const eligibleFindings = filterEligibleFindings(ineligibleFindings);
const prompt = buildSynthesisPrompt(canonicalGraph, eligibleFindings);
expect(prompt).toContain("Eligible Findings");
// The excluded proposition must not appear because eligible findings is empty
expect(eligibleFindings).toHaveLength(0);
// ── C. unknown excluded ──────────────────────────────────
it("unknown node content does NOT appear anywhere in synthesis", () => {
const prompt = buildSynthesisPrompt(fullScenario, []);
expect(prompt).not.toContain("Baseline period denominator volume");
expect(prompt).not.toContain('status":"unknown"');
expect(prompt).not.toContain("question about the baseline");
});
it("includes provisional (null disposition) findings", () => {
const findings = [makeFinding({ userDisposition: null })];
const prompt = buildSynthesisPrompt(canonicalGraph, findings);
expect(prompt).toContain("Provisional Findings");
expect(prompt).toContain("Supplier delays caused production halts");
// ── D. provisional excluded ──────────────────────────────
it("provisional hypothesis does NOT appear as CU evidence", () => {
const prompt = buildSynthesisPrompt(fullScenario, []);
expect(prompt).not.toContain("Product mix may explain");
expect(prompt).not.toContain('status":"provisional"');
});
it("includes confirmed (agree) findings", () => {
const findings = [makeFinding({ userDisposition: "agree" })];
const prompt = buildSynthesisPrompt(canonicalGraph, findings);
// ── E. resolved-question text excluded ───────────────────
it("resolved node label/description does NOT appear as CU evidence", () => {
const prompt = buildSynthesisPrompt(fullScenario, []);
expect(prompt).not.toContain("Time period over which these percentage changes occurred");
expect(prompt).not.toContain('status":"resolved"');
expect(prompt).not.toContain("Resolved question");
});
// ── F. learned Finding retained ──────────────────────────
it("eligible Finding appears in synthesis — proves we exclude resolved question while retaining the answer", () => {
const prompt = buildSynthesisPrompt(fullScenario, [workingFinding()]);
expect(prompt).toContain("The reported percentage changes correspond to the last financial quarter.");
expect(prompt).not.toContain("Time period over which these percentage changes occurred");
});
// ── G. control/reasoning content excluded ────────────────
it("control/reasoning sentinels are absent from synthesis prompt", () => {
const prompt = buildSynthesisPrompt(fullScenario, []);
expect(prompt).not.toContain("currentSummary");
expect(prompt).not.toContain("Sentinel: currentSummary must not appear in synthesis");
expect(prompt).not.toContain("reasoningState");
expect(prompt).not.toContain("activeUnknownNodeId");
expect(prompt).not.toContain('"n3"'); // active unknown node id sentinel
expect(prompt).not.toContain("resolvedNodeIds");
expect(prompt).not.toContain('"n4"'); // resolved node id sentinel
});
// ── H. centralStatement retained as framing ─────────────
it("centralStatement remains available as framing context", () => {
const prompt = buildSynthesisPrompt(fullScenario, []);
expect(prompt).toContain("Complaints increased by 35% while production increased by 40%");
});
// ── I. node.proposition dead read removed ────────────────
it("prompt does NOT depend on a proposition property on SituationGraph nodes", () => {
const noPropositionGraph = {
centralStatement: "Only framing.",
nodes: [
{ id: "a1", label: "Label only node", description: "No proposition field", kind: "observation", status: "known", confidence: "medium" },
],
};
const prompt = buildSynthesisPrompt(noPropositionGraph, []);
expect(prompt).toContain("Only framing.");
expect(prompt).toContain("Label only node");
// The projection must work correctly even when nodes lack a 'propertion' field entirely.
});
// ── Structural correctness of the text-formatted evidence section ─
it("evidence section contains Known Facts and Supported Inferences headers", () => {
const prompt = buildSynthesisPrompt(fullScenario, []);
expect(prompt).toContain("Known Facts:");
expect(prompt).toContain("Supported Inferences:");
});
it("evidence section does not contain raw graph fields (edges, resolvedNodeIds, activeUnknownNodeId, reasoningState)", () => {
const prompt = buildSynthesisPrompt(fullScenario, []);
// These are all control/structural graph fields that must not appear in the evidence section
expect(prompt).not.toContain("edges");
expect(prompt).not.toContain("resolvedNodeIds");
expect(prompt).not.toContain("activeUnknownNodeId");
expect(prompt).not.toContain("reasoningState");
});
it("node representation includes all canonical fields — kind, label, status, value, unit", () => {
const prompt = buildSynthesisPrompt(fullScenario, []);
// Each node line contains: kind=..., label="...", value=... [status]
expect(prompt).toContain("kind=metric");
expect(prompt).toContain('label="Complaint count increased by 35%"');
expect(prompt).toContain("[known]");
expect(prompt).toContain("[supported]");
expect(prompt).toContain("value=35 (");
expect(prompt).toContain("Primary metric baseline");
});
it("provider sees structured evidence section header", () => {
const prompt = buildSynthesisPrompt(fullScenario, []);
expect(prompt).toContain("Provider-Active Evidence:");
expect(prompt).not.toContain("Canonical Situation Graph");
});
// ── Edge case: empty graph (no known/supported nodes) ────
it("handles graph with no evidence nodes gracefully", () => {
const prompt = buildSynthesisPrompt(emptyGraph, []);
expect(prompt).not.toContain("(no evidence)");
expect(prompt).toContain("Complaints increased by 35% while production increased by 40%");
});
// ── Edge case: only supported, no known ──────────────────
it("handles graph with only supported nodes", () => {
const supportedOnly = {
centralStatement: "Metric drift detected.",
nodes: [
{ id: "s1", label: "Drift exceeds threshold", description: "Supported inference", kind: "metric", status: "supported", confidence: "low" },
],
};
const prompt = buildSynthesisPrompt(supportedOnly, []);
expect(prompt).toContain("Drift exceeds threshold");
});
// ── Finding integration tests ────────────────────────────
it("agreed Finding appears in synthesis with Correct Evidence label", () => {
const prompt = buildSynthesisPrompt(fullScenario, [agreedFinding()]);
expect(prompt).toContain("The percentage increase in complaints exceeds the threshold.");
expect(prompt).toContain("Confirmed Evidence");
});
it("handles zero eligible Findings — synthesis still proceeds from graph alone", () => {
const prompt = buildSynthesisPrompt(canonicalGraph, []);
expect(prompt).toContain("(none)");
// Must still contain graph content
expect(prompt).toContain("Canonical Situation Graph");
it("working premise Finding appears with Working Premises label", () => {
const prompt = buildSynthesisPrompt(fullScenario, [workingFinding()]);
expect(prompt).toContain("The reported percentage changes correspond to the last financial quarter.");
expect(prompt).toContain("Working Premises");
});
it("prompt contains fresh-synthesis instructions (not append semantics)", () => {
const prompt = buildSynthesisPrompt(canonicalGraph, []);
expect(prompt).toContain("FRESH synthesis");
expect(prompt).toContain("Do NOT treat any previous Current Understanding as input");
expect(prompt).toContain("Do NOT append to prior summaries");
// ── Immutability ─────────────────────────────────────────
it("graph structurally unchanged after synthesis prompt build", () => {
const graphSnapshot = JSON.parse(JSON.stringify(fullScenario));
buildSynthesisPrompt(fullScenario, []);
expect(JSON.stringify(fullScenario)).toBe(JSON.stringify(graphSnapshot));
});
it("prompt does NOT request graph mutations or Finding mutations", () => {
const prompt = buildSynthesisPrompt(canonicalGraph, []);
expect(prompt).not.toMatch(/change\s+selectedQuestion/i);
it("findings structurally unchanged after synthesis prompt build", () => {
const findings = [workingFinding()];
const snapshot = JSON.parse(JSON.stringify(findings));
buildSynthesisPrompt(fullScenario, findings);
expect(JSON.stringify(findings)).toBe(JSON.stringify(snapshot));
});
// ── Immutability during full seam ────────────────────────
it("graph and findings unchanged after synthesizeCurrentUnderstanding", async () => {
const graphSnapshot = JSON.parse(JSON.stringify(fullScenario));
const findings = [workingFinding()];
const findingsSnapshot = JSON.parse(JSON.stringify(findings));
const fake = makeFakeProvider();
await synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings },
{ provider: fake }
);
expect(JSON.stringify(fullScenario)).toBe(JSON.stringify(graphSnapshot));
expect(JSON.stringify(findings)).toBe(JSON.stringify(findingsSnapshot));
});
});
// ── Output validation tests ─────────────────────────────────
// ── Prompt authority rule verification ─────────────────────
describe("buildSynthesisPrompt — prompt semantics", () => {
it("prompt contains evidence-authority boundary rule (not open questions)", () => {
const prompt = buildSynthesisPrompt(fullScenario, []);
expect(prompt).toContain("Current Understanding describes only established or supported understanding");
expect(prompt).toContain("Do not introduce or describe open questions, unresolved uncertainties, assumptions, provisional hypotheses");
});
it("prompt clarifies centralStatement is framing context only", () => {
const prompt = buildSynthesisPrompt(fullScenario, []);
expect(prompt).toContain("Framing");
// Must explicitly say the centralStatement is not independent evidence
expect(prompt).toContain("framing context, not independent evidence");
});
it("prompt does NOT include edges in the evidence section", () => {
const prompt = buildSynthesisPrompt(fullScenario, []);
// Edges should not be mentioned in the structured evidence projection
expect(prompt).not.toContain("Edge");
expect(prompt).not.toMatch(/relationship:.*compares_with/);
});
});
// ── Output validation tests (unchanged) ────────────────────
describe("validateSynthesisResponse", () => {
it("accepts valid narrative JSON object", () => {
@@ -199,153 +353,120 @@ describe("validateSynthesisResponse", () => {
});
it("rejects null input", () => {
const result = validateSynthesisResponse(null);
expect(result.valid).toBe(false);
expect(validateSynthesisResponse(null).valid).toBe(false);
});
it("rejects undefined input", () => {
const result = validateSynthesisResponse(undefined);
expect(result.valid).toBe(false);
expect(validateSynthesisResponse(undefined).valid).toBe(false);
});
});
// ── Domain function tests ───────────────────────────────────
// ── Domain function tests (full seam) ──────────────────────
describe("synthesizeCurrentUnderstanding — full seam", () => {
it("null disposition finding → included in synthesis", async () => {
const fake = makeFakeProvider();
const result = await synthesizeCurrentUnderstanding(
{ situationGraph: canonicalGraph, findings: [makeFinding({ userDisposition: null })] },
{ provider: fake }
);
expect(result.currentUnderstanding).toBe("Synthesized output");
expect(fake.generateReconstruction).toHaveBeenCalledTimes(1);
const prompt = fake.generateReconstruction.mock.calls[0][0];
expect(prompt).toContain("Supplier delays caused production halts");
});
// ── Evidence inclusion/exclusion at seam level ───────────
it("agree disposition finding → included in synthesis", async () => {
const fake = makeFakeProvider();
const result = await synthesizeCurrentUnderstanding(
{ situationGraph: canonicalGraph, findings: [makeFinding({ userDisposition: "agree" })] },
{ provider: fake }
);
expect(result.currentUnderstanding).toBe("Synthesized output");
});
it("not_quite disposition finding → excluded from synthesis", async () => {
const fake = makeFakeProvider();
const result = await synthesizeCurrentUnderstanding(
{ situationGraph: canonicalGraph, findings: [makeFinding({ userDisposition: "not_quite" })] },
{ provider: fake }
);
expect(result.currentUnderstanding).toBe("Synthesized output");
const prompt = fake.generateReconstruction.mock.calls[0][0];
// The not_quite proposition must NOT appear because it was excluded by eligibility
expect(prompt).toContain("(none)");
expect(prompt).not.toContain("Supplier delays caused production halts");
});
it("not_relevant disposition finding → excluded from synthesis", async () => {
it("known node content flows to provider via synthesis prompt", async () => {
const fake = makeFakeProvider();
await synthesizeCurrentUnderstanding(
{ situationGraph: canonicalGraph, findings: [makeFinding({ userDisposition: "not_relevant" })] },
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
);
const prompt = fake.generateReconstruction.mock.calls[0][0];
expect(prompt).toContain("(none)");
expect(prompt).toContain("Complaint count increased by 35%");
expect(prompt).toContain("[known]");
});
it("rejected evaluation → excluded from synthesis", async () => {
it("supported node content flows to provider via synthesis prompt", async () => {
const fake = makeFakeProvider();
await synthesizeCurrentUnderstanding(
{ situationGraph: canonicalGraph, findings: [makeFinding({ evaluation: "rejected" })] },
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
);
const prompt = fake.generateReconstruction.mock.calls[0][0];
expect(prompt).toContain("(none)");
expect(prompt).toContain("Production count increased by 40%");
expect(prompt).toContain("[supported]");
});
it("zero eligible Findings → synthesis succeeds from graph alone", async () => {
it("unknown nodes do NOT appear in provider-visible synthesis", async () => {
const fake = makeFakeProvider();
const result = await synthesizeCurrentUnderstanding(
{ situationGraph: canonicalGraph, findings: [] },
await synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
);
expect(result.currentUnderstanding).toBe("Synthesized output");
const prompt = fake.generateReconstruction.mock.calls[0][0];
// Must still contain graph content
expect(prompt).toContain("Canonical Situation Graph");
expect(prompt).toContain("Node(n1)");
expect(prompt).not.toContain("Baseline period denominator volume");
});
it("fake provider returns valid narrative → { currentUnderstanding }", async () => {
it("provisional hypothesis does NOT appear in synthesis prompt", async () => {
const fake = makeFakeProvider();
const result = await synthesizeCurrentUnderstanding(
{ situationGraph: canonicalGraph, findings: [makeFinding({ userDisposition: "agree" })] },
await synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
);
expect(result).toEqual({ currentUnderstanding: "Synthesized output" });
const prompt = fake.generateReconstruction.mock.calls[0][0];
expect(prompt).not.toContain("Product mix may explain");
});
it("fake provider returns empty narrative → rejected", async () => {
const fake = makeFakeProvider(async () => JSON.stringify({ currentUnderstanding: "" }));
await expect(
synthesizeCurrentUnderstanding(
{ situationGraph: canonicalGraph, findings: [] },
{ provider: fake }
)
).rejects.toThrow(/Synthesis validation failed/);
it("resolved-question text does NOT appear in synthesis prompt", async () => {
const fake = makeFakeProvider();
await synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
);
const prompt = fake.generateReconstruction.mock.calls[0][0];
expect(prompt).not.toContain("Time period over which these percentage changes occurred");
});
it("fake provider returns malformed JSON → rejected", async () => {
const fake = makeFakeProvider(async () => "{ not valid json");
await expect(
synthesizeCurrentUnderstanding(
{ situationGraph: canonicalGraph, findings: [] },
{ provider: fake }
)
).rejects.toThrow(/Synthesis validation failed/);
it("eligible Finding appears in synthesis", async () => {
const fake = makeFakeProvider();
await synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings: [workingFinding()] },
{ provider: fake }
);
const prompt = fake.generateReconstruction.mock.calls[0][0];
expect(prompt).toContain("The reported percentage changes correspond to the last financial quarter.");
});
it("fake provider returns object without currentUnderstanding → rejected", async () => {
const fake = makeFakeProvider(async () => JSON.stringify({ wrongField: "value" }));
await expect(
synthesizeCurrentUnderstanding(
{ situationGraph: canonicalGraph, findings: [] },
{ provider: fake }
)
).rejects.toThrow(/Synthesis validation failed/);
it("control/reasoning sentinels excluded from synthesis", async () => {
const fake = makeFakeProvider();
await synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
);
const prompt = fake.generateReconstruction.mock.calls[0][0];
expect(prompt).not.toContain("currentSummary");
expect(prompt).not.toContain("reasoningState");
});
// ── Immutability tests ────────────────────────────────────
// ── Immutability ─────────────────────────────────────────
it("graph structurally unchanged after synthesis", async () => {
const graphSnapshot = JSON.parse(JSON.stringify(canonicalGraph));
const graphSnapshot = JSON.parse(JSON.stringify(fullScenario));
const fake = makeFakeProvider();
await synthesizeCurrentUnderstanding(
{ situationGraph: canonicalGraph, findings: [] },
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
);
expect(JSON.stringify(canonicalGraph)).toBe(JSON.stringify(graphSnapshot));
expect(JSON.stringify(fullScenario)).toBe(JSON.stringify(graphSnapshot));
});
it("findings structurally unchanged after synthesis", async () => {
const findings = [makeFinding({ userDisposition: "agree" })];
const findings = [workingFinding()];
const findingsSnapshot = JSON.parse(JSON.stringify(findings));
const fake = makeFakeProvider();
await synthesizeCurrentUnderstanding(
{ situationGraph: canonicalGraph, findings },
{ situationGraph: fullScenario, findings },
{ provider: fake }
);
expect(JSON.stringify(findings)).toBe(JSON.stringify(findingsSnapshot));
});
// ── Input validation tests ────────────────────────────────
// ── Input validation (unchanged) ────────────────────────
it("missing situationGraph → throws 400", async () => {
await expect(
@@ -361,7 +482,7 @@ describe("synthesizeCurrentUnderstanding — full seam", () => {
it("findings as non-array → throws 400", async () => {
await expect(
synthesizeCurrentUnderstanding({ situationGraph: canonicalGraph, findings: "string" }, { provider: makeFakeProvider() })
synthesizeCurrentUnderstanding({ situationGraph: fullScenario, findings: "string" }, { provider: makeFakeProvider() })
).rejects.toThrow(/findings must be an array/);
});
@@ -369,47 +490,36 @@ describe("synthesizeCurrentUnderstanding — full seam", () => {
const fake = makeFakeProvider(async () => { throw new Error("provider down"); });
await expect(
synthesizeCurrentUnderstanding(
{ situationGraph: canonicalGraph, findings: [] },
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
)
).rejects.toThrow(/Synthesis provider call failed|provider down/);
});
it("no provider provided — falls through to getProvider() which needs env vars", async () => {
it("no provider provided — falls through to getProvider()", async () => {
await expect(
synthesizeCurrentUnderstanding(
{ situationGraph: canonicalGraph, findings: [] },
{ situationGraph: fullScenario, findings: [] },
{}
)
).rejects.toThrow(/OLLAMA_BASE_URL/);
});
// ── Configured model resolution (unchanged) ─────────────
// ── Configured model resolution ──────────────────────────
it("default synthesis path resolves configured modelName — provider receives non-null", async () => {
it("default synthesis path resolves configured modelName", async () => {
const fake = makeFakeProvider();
// Stub the configured model so test does not depend on dev-machine .env.local
const savedModel = process.env.OLLAMA_MODEL;
process.env.OLLAMA_MODEL = "configured-model-v0.50";
try {
const result = await synthesizeCurrentUnderstanding(
{ situationGraph: canonicalGraph, findings: [] },
await synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
);
expect(result.currentUnderstanding).toBe("Synthesized output");
// KEY ASSERTION: configured model must flow to provider
const receivedModel = fake.generateReconstruction.mock.calls[0][1];
expect(receivedModel).toBeDefined();
expect(receivedModel).not.toBeNull();
expect(typeof receivedModel).toBe("string");
expect(receivedModel.length).toBeGreaterThan(0);
} finally {
// Restore original env value (may be undefined)
if (savedModel == null) {
delete process.env.OLLAMA_MODEL;
} else {
@@ -418,37 +528,96 @@ describe("synthesizeCurrentUnderstanding — full seam", () => {
}
});
it("explicit modelName dependency overrides default — provider receives injected model", async () => {
it("explicit modelName dependency overrides default", async () => {
const fake = makeFakeProvider();
await synthesizeCurrentUnderstanding(
{ situationGraph: canonicalGraph, findings: [] },
{ situationGraph: fullScenario, findings: [] },
{ provider: fake, modelName: "test-model-v0.50" }
);
expect(fake.generateReconstruction.mock.calls[0][1]).toBe("test-model-v0.50");
});
// ── Provider sees full graph, not just centralStatement ───
// ── Provider sees evidence projection, not raw graph ─────
it("provider receives full canonical graph content (nodes + edges + centralStatement)", async () => {
it("provider receives structured evidence projection (not full graph dump)", async () => {
const fake = makeFakeProvider();
await synthesizeCurrentUnderstanding(
{ situationGraph: canonicalGraph, findings: [] },
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
);
const prompt = fake.generateReconstruction.mock.calls[0][0];
// Must contain node content (not just centralStatement)
expect(prompt).toContain("Node(n1)");
expect(prompt).toContain("Q1 revenue was stable");
expect(prompt).toContain("Node(n2)");
expect(prompt).toContain("Supplier A failed deliveries in May");
// Must contain edge content
expect(prompt).toContain("Edge(n2 → n3");
expect(prompt).toContain("causal");
// Must contain centralStatement value
expect(prompt).toContain("Revenue dropped 30%");
// Must instruct about fresh synthesis (not previous CU)
expect(prompt).toContain("Do NOT treat any previous Current Understanding as input");
// Evidence section must be structured, not a raw JSON dump of the whole graph
expect(prompt).toContain("Provider-Active Evidence");
expect(prompt).not.toContain("Canonical Situation Graph");
});
it("provider receives centralStatement as framing context", async () => {
const fake = makeFakeProvider();
await synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
);
const prompt = fake.generateReconstruction.mock.calls[0][0];
expect(prompt).toContain("Complaints increased by 35%");
expect(prompt).toContain("Framing");
});
it("provider receives centralStatement framing for no-evidence graph", async () => {
const fake = makeFakeProvider();
await synthesizeCurrentUnderstanding(
{ situationGraph: emptyGraph, findings: [] },
{ provider: fake }
);
const prompt = fake.generateReconstruction.mock.calls[0][0];
// Even with no evidence nodes, centralStatement framing flows through
expect(prompt).toContain("Central Statement");
});
it("null/undefined findings → synthesis proceeds with empty evidence", async () => {
const fake = makeFakeProvider();
await synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings: null },
{ provider: fake }
);
expect(fake.generateReconstruction).toHaveBeenCalledTimes(1);
});
it("fake provider returns valid narrative → { currentUnderstanding }", async () => {
const fake = makeFakeProvider();
const result = await synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings: [workingFinding()] },
{ provider: fake }
);
expect(result).toEqual({ currentUnderstanding: "Synthesized output" });
});
it("fake provider returns empty narrative → rejected", async () => {
const fake = makeFakeProvider(async () => JSON.stringify({ currentUnderstanding: "" }));
await expect(
synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
)
).rejects.toThrow(/Synthesis validation failed/);
});
it("fake provider returns malformed JSON → rejected", async () => {
const fake = makeFakeProvider(async () => "{ not valid json");
await expect(
synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
)
).rejects.toThrow(/Synthesis validation failed/);
});
it("fake provider returns object without currentUnderstanding → rejected", async () => {
const fake = makeFakeProvider(async () => JSON.stringify({ wrongField: "value" }));
await expect(
synthesizeCurrentUnderstanding(
{ situationGraph: fullScenario, findings: [] },
{ provider: fake }
)
).rejects.toThrow(/Synthesis validation failed/);
});
});