robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: evaluate deterministic cross-branch corroboration

Browse Source
This commit is contained in:
robbond
2026-08-03 07:19:20 +01:00
parent 1d64144e01
commit b2ffc54964
6 changed files with 547 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/v0.6-atomicity-experiment.md
+34
View File
@@ -149,6 +149,40 @@ Deterministic propagation rules now enforce:
- status changes do not raise confidence on their own
- parent resolution still requires the separate completion rule
## Cross-branch corroboration
The next confidence experiment adds deterministic branch interaction checks without changing the graph model.
The engine now distinguishes between:
- **multiple evidence**: more than one branch exists
- **independent corroboration**: distinct resolved branches support the same parent without sharing the same evidence key
- **duplicate evidence**: the same evidence key appears through multiple branches and must not be double-counted
- **conflicting evidence**: branches support incompatible positions, such as `recognised correctly` vs `recognised incorrectly`
Deterministic branch rules:
- corroboration only counts when branches are distinct and their evidence sources differ
- duplicate evidence groups never count as corroboration
- conflicts cap conclusion confidence and prevent a higher confidence upgrade
- independent branches remain interaction-neutral
Additional diagnostics now expose:
- `corroboratingBranchCount`
- `conflictingBranchCount`
- `duplicateEvidenceCount`
- `independentBranchCount`
- `interactionSummary`
- `confidenceAdjustmentReason`
Observed effect:
- independent corroboration can raise `evidenceConfidence`
- duplicate evidence produces no extra confidence increase
- conflicting evidence lowers or caps `conclusionConfidence`
- completeness rules still dominate whether a parent may become highly justified
Example progression:
- parent before: `unknown`, `medium`
lib/graph/apply-proposal.js
+179 -4
View File
@@ -463,6 +463,129 @@ function confidenceFromAssessment(assessment) {
return assessment?.conclusionConfidence ?? "medium";
}
function unique(values = []) {
return [...new Set(values.filter(Boolean))];
}
function branchEvidenceKeys(node) {
return unique([...(node?.evidenceIds || []), node?.value]);
}
function sharedMeaningfulTokens(aText, bText) {
const stop = new Set([
"the",
"and",
"for",
"that",
"this",
"with",
"from",
"because",
"need",
"unknown",
"possible",
]);
const a = splitSemanticTokens(aText).filter((token) => !stop.has(token));
const b = splitSemanticTokens(bText).filter((token) => !stop.has(token));
return [...new Set(a.filter((token) => b.includes(token)))];
}
function branchConflictSignature(node) {
return normaliseText(
`${node?.label || ""} ${node?.description || ""} ${node?.value || ""}`,
);
}
function branchesConflict(aNode, bNode) {
const aText = branchConflictSignature(aNode);
const bText = branchConflictSignature(bNode);
const oppositePolarity =
(aText.includes("correctly") && bText.includes("incorrectly")) ||
(aText.includes("incorrectly") && bText.includes("correctly")) ||
aNode?.status === "contradicted" ||
bNode?.status === "contradicted";
if (!oppositePolarity) return false;
return sharedMeaningfulTokens(aText, bText).length >= 2;
}
export function evaluateBranchInteractions({ parentNode, graph }) {
const directBranches = findDirectChildUnknowns(graph, parentNode.id).filter(
(node) => ["resolved", "provisional", "contradicted"].includes(node.status),
);
const duplicateEvidenceGroups = [];
const conflictingBranches = [];
const corroboratingBranches = [];
const duplicateBranchIds = new Set();
const conflictingBranchIds = new Set();
const evidenceGroups = new Map();
for (const branch of directBranches) {
for (const evidenceKey of branchEvidenceKeys(branch)) {
const ids = evidenceGroups.get(evidenceKey) || [];
ids.push(branch.id);
evidenceGroups.set(evidenceKey, ids);
}
}
for (const [evidenceKey, branchIds] of evidenceGroups.entries()) {
if (branchIds.length > 1) {
duplicateEvidenceGroups.push({
evidenceKey,
branchIds: unique(branchIds),
});
for (const id of branchIds) duplicateBranchIds.add(id);
}
}
for (let index = 0; index < directBranches.length; index += 1) {
for (let inner = index + 1; inner < directBranches.length; inner += 1) {
const aNode = directBranches[index];
const bNode = directBranches[inner];
if (branchesConflict(aNode, bNode)) {
conflictingBranches.push([aNode.id, bNode.id]);
conflictingBranchIds.add(aNode.id);
conflictingBranchIds.add(bNode.id);
continue;
}
const aEvidence = branchEvidenceKeys(aNode);
const bEvidence = branchEvidenceKeys(bNode);
const sharesEvidence = aEvidence.some((key) => bEvidence.includes(key));
if (
!sharesEvidence &&
aNode.status === "resolved" &&
bNode.status === "resolved"
) {
corroboratingBranches.push([aNode.id, bNode.id]);
}
}
}
const interactionBranchIds = new Set([
...duplicateBranchIds,
...conflictingBranchIds,
...corroboratingBranches.flat(),
]);
const independentBranches = directBranches
.map((branch) => branch.id)
.filter((id) => !interactionBranchIds.has(id));
return {
corroboratingBranches,
conflictingBranches,
duplicateEvidenceGroups,
independentBranches,
interactionSummary: {
corroboratingBranchCount: corroboratingBranches.length,
conflictingBranchCount: conflictingBranches.length,
duplicateEvidenceCount: duplicateEvidenceGroups.length,
independentBranchCount: independentBranches.length,
},
};
}
function upsertProposalNodeUpdate(proposalSnapshot, update) {
const existing = proposalSnapshot.updatedNodes.find(
(candidate) => candidate.nodeId === update.nodeId,
@@ -585,6 +708,13 @@ function computeParentProgressState(graph, parentNode) {
(child) => child.status !== "resolved",
).length;
const beforeAssessment = getNodeConfidenceAssessment(parentNode);
const interactions = evaluateBranchInteractions({ parentNode, graph });
const corroborationCount =
interactions.interactionSummary.corroboratingBranchCount;
const duplicateEvidenceCount =
interactions.interactionSummary.duplicateEvidenceCount;
const conflictingBranchCount =
interactions.interactionSummary.conflictingBranchCount;
if (totalChildren === 0) {
const nextAssessment = {
@@ -625,18 +755,32 @@ function computeParentProgressState(graph, parentNode) {
confidenceCapReason = "no_resolved_direct_children";
} else if (resolvedCount < totalChildren) {
nextAssessment = {
evidenceConfidence: "high",
evidenceConfidence: corroborationCount > 0 ? "high" : "medium",
completenessStatus: "partial",
conclusionConfidence: "medium",
};
confidenceCapReason = "unresolved_direct_children_cap_conclusion";
confidenceCapReason =
conflictingBranchCount > 0
? "conflicting_branches_cap_conclusion"
: duplicateEvidenceCount > 0
? "duplicate_evidence_no_extra_confidence"
: corroborationCount > 0
? "independent_corroboration_with_incomplete_parent"
: "unresolved_direct_children_cap_conclusion";
} else {
nextAssessment = {
evidenceConfidence: "high",
completenessStatus: "complete",
conclusionConfidence: "high",
conclusionConfidence: conflictingBranchCount > 0 ? "low" : "high",
};
confidenceCapReason = null;
confidenceCapReason =
conflictingBranchCount > 0
? "conflicting_branches_cap_conclusion"
: duplicateEvidenceCount > 0
? "duplicate_evidence_no_extra_confidence"
: corroborationCount > 0
? "independent_corroboration_supported_conclusion"
: null;
}
if (resolvedChildren.length === totalChildren) {
@@ -652,6 +796,7 @@ function computeParentProgressState(graph, parentNode) {
resolvedDirectChildren: resolvedCount,
unresolvedDirectChildren: unresolvedCount,
contradictoryDirectChildren: contradictoryChildren.length,
branchInteractions: interactions,
confidenceCapReason,
reason:
"All direct child unknowns are resolved, so the parent can now resolve deterministically.",
@@ -671,6 +816,7 @@ function computeParentProgressState(graph, parentNode) {
resolvedDirectChildren: resolvedCount,
unresolvedDirectChildren: unresolvedCount,
contradictoryDirectChildren: contradictoryChildren.length,
branchInteractions: interactions,
confidenceCapReason,
reason:
"At least one direct child has been progressed, so the parent becomes provisional but remains unresolved until all direct children are resolved.",
@@ -689,6 +835,7 @@ function computeParentProgressState(graph, parentNode) {
resolvedDirectChildren: resolvedCount,
unresolvedDirectChildren: unresolvedCount,
contradictoryDirectChildren: contradictoryChildren.length,
branchInteractions: interactions,
confidenceCapReason,
reason: "No direct child progress exists yet for the parent.",
};
@@ -807,6 +954,19 @@ export function propagateResolvedChildEvidence({
resolvedDirectChildren: progressState.resolvedDirectChildren,
unresolvedDirectChildren: progressState.unresolvedDirectChildren,
contradictoryDirectChildren: progressState.contradictoryDirectChildren,
corroboratingBranchCount:
progressState.branchInteractions.interactionSummary
.corroboratingBranchCount,
conflictingBranchCount:
progressState.branchInteractions.interactionSummary
.conflictingBranchCount,
duplicateEvidenceCount:
progressState.branchInteractions.interactionSummary
.duplicateEvidenceCount,
independentBranchCount:
progressState.branchInteractions.interactionSummary
.independentBranchCount,
interactionSummary: progressState.branchInteractions.interactionSummary,
confidenceCapReason: progressState.confidenceCapReason,
parentResolved: progressState.parentResolved,
reason: progressState.reason,
@@ -846,6 +1006,11 @@ export function propagateResolvedChildEvidence({
resolvedDirectChildren: firstEvent?.resolvedDirectChildren ?? 0,
unresolvedDirectChildren: firstEvent?.unresolvedDirectChildren ?? 0,
contradictoryDirectChildren: firstEvent?.contradictoryDirectChildren ?? 0,
corroboratingBranchCount: firstEvent?.corroboratingBranchCount ?? 0,
conflictingBranchCount: firstEvent?.conflictingBranchCount ?? 0,
duplicateEvidenceCount: firstEvent?.duplicateEvidenceCount ?? 0,
independentBranchCount: firstEvent?.independentBranchCount ?? 0,
interactionSummary: firstEvent?.interactionSummary ?? null,
confidenceCapReason: firstEvent?.confidenceCapReason ?? null,
ancestorPropagationStoppedReason,
affectedAncestorIds: [...affectedAncestorIds],
@@ -1915,6 +2080,11 @@ export function applyValidatedProposal({
const affectedAncestorIds = propagationResult.affectedAncestorIds;
const nextSelectedSibling = propagationResult.nextSelectedSibling;
const parentResolved = propagationResult.parentResolved;
const corroboratingBranchCount = propagationResult.corroboratingBranchCount;
const conflictingBranchCount = propagationResult.conflictingBranchCount;
const duplicateEvidenceCount = propagationResult.duplicateEvidenceCount;
const independentBranchCount = propagationResult.independentBranchCount;
const interactionSummary = propagationResult.interactionSummary;
const propagationReason = propagationResult.reason;
if (
@@ -2048,6 +2218,11 @@ export function applyValidatedProposal({
resolvedDirectChildren,
unresolvedDirectChildren,
contradictoryDirectChildren,
corroboratingBranchCount,
conflictingBranchCount,
duplicateEvidenceCount,
independentBranchCount,
interactionSummary,
confidenceCapReason,
ancestorPropagationStoppedReason,
affectedAncestorIds,
lib/graph/orchestrator.js
+25
View File
@@ -119,6 +119,11 @@ function buildUpdateDiagnostics({
resolvedDirectChildren,
unresolvedDirectChildren,
contradictoryDirectChildren,
corroboratingBranchCount,
conflictingBranchCount,
duplicateEvidenceCount,
independentBranchCount,
interactionSummary,
confidenceCapReason,
ancestorPropagationStoppedReason,
affectedAncestorIds,
@@ -183,6 +188,11 @@ function buildUpdateDiagnostics({
resolvedDirectChildren: resolvedDirectChildren ?? 0,
unresolvedDirectChildren: unresolvedDirectChildren ?? 0,
contradictoryDirectChildren: contradictoryDirectChildren ?? 0,
corroboratingBranchCount: corroboratingBranchCount ?? 0,
conflictingBranchCount: conflictingBranchCount ?? 0,
duplicateEvidenceCount: duplicateEvidenceCount ?? 0,
independentBranchCount: independentBranchCount ?? 0,
interactionSummary: interactionSummary ?? null,
confidenceCapReason: confidenceCapReason ?? null,
ancestorPropagationStoppedReason: ancestorPropagationStoppedReason ?? null,
affectedAncestorIds: affectedAncestorIds ?? [],
@@ -471,6 +481,11 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
resolvedDirectChildren: 0,
unresolvedDirectChildren: 0,
contradictoryDirectChildren: 0,
corroboratingBranchCount: 0,
conflictingBranchCount: 0,
duplicateEvidenceCount: 0,
independentBranchCount: 0,
interactionSummary: null,
confidenceCapReason: null,
ancestorPropagationStoppedReason: null,
affectedAncestorIds: [],
@@ -552,6 +567,11 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
unresolvedDirectChildren: applicationResult.unresolvedDirectChildren,
contradictoryDirectChildren:
applicationResult.contradictoryDirectChildren,
corroboratingBranchCount: applicationResult.corroboratingBranchCount,
conflictingBranchCount: applicationResult.conflictingBranchCount,
duplicateEvidenceCount: applicationResult.duplicateEvidenceCount,
independentBranchCount: applicationResult.independentBranchCount,
interactionSummary: applicationResult.interactionSummary,
confidenceCapReason: applicationResult.confidenceCapReason,
ancestorPropagationStoppedReason:
applicationResult.ancestorPropagationStoppedReason,
@@ -616,6 +636,11 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
resolvedDirectChildren: 0,
unresolvedDirectChildren: 0,
contradictoryDirectChildren: 0,
corroboratingBranchCount: 0,
conflictingBranchCount: 0,
duplicateEvidenceCount: 0,
independentBranchCount: 0,
interactionSummary: null,
confidenceCapReason: null,
ancestorPropagationStoppedReason: null,
affectedAncestorIds: [],
tests/graph/confidence-propagation.test.js
+1 -1
View File
@@ -127,7 +127,7 @@ describe("confidence propagation", () => {
expect(parent.status).toBe("provisional");
expect(parent.confidence).toBe("medium");
expect(parent.confidenceAssessment).toEqual({
evidenceConfidence: "high",
evidenceConfidence: "medium",
completenessStatus: "partial",
conclusionConfidence: "medium",
});
tests/graph/cross-branch-corroboration.test.js
+306
View File
@@ -0,0 +1,306 @@
import { describe, expect, it } from "vitest";
import {
applyValidatedProposal,
evaluateBranchInteractions,
} from "@/lib/graph/apply-proposal.js";
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
function makeParentWithBranches(children) {
const parent = makeNode({
id: "n-parent",
label: "Explanation for why revenue increased while cash fell",
description:
"Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
kind: "unknown",
status: "unknown",
confidence: "medium",
});
return makeGraph({
centralStatement: "Revenue increased while cash fell.",
nodes: [
parent,
...children.map((child) => ({ ...child, parentId: parent.id })),
],
edges: children.map((child, index) =>
makeEdge({
id: `e-${index + 1}`,
fromNodeId: child.id,
toNodeId: parent.id,
relationship: "depends_on",
description: `${child.label} feeds the parent.`,
}),
),
activeUnknownNodeId: children[0]?.id ?? null,
resolvedNodeIds: [],
currentSummary: "cross-branch corroboration fixture",
});
}
function makeResolvedChild(id, label, value, extra = {}) {
return makeNode({
id,
label,
description: label,
kind: "unknown",
status: "resolved",
confidence: "medium",
value,
evidenceIds: extra.evidenceIds ?? [],
});
}
function makeUnknownBranch(id, label, description, extra = {}) {
return makeNode({
id,
label,
description,
kind: "unknown",
status: extra.status ?? "unknown",
confidence: extra.confidence ?? "medium",
evidenceIds: extra.evidenceIds ?? [],
value: extra.value ?? null,
});
}
describe("evaluateBranchInteractions", () => {
it("detects corroborating independent branches", () => {
const graph = makeParentWithBranches([
makeResolvedChild("n-a", "Debtor balance increased", "bank-statement-a", {
evidenceIds: ["bank-statement-a"],
}),
makeResolvedChild(
"n-b",
"Cash receipts were delayed",
"receipts-ledger-b",
{ evidenceIds: ["receipts-ledger-b"] },
),
]);
const parentNode = graph.nodes.find((node) => node.id === "n-parent");
const result = evaluateBranchInteractions({ parentNode, graph });
expect(result.interactionSummary.corroboratingBranchCount).toBe(1);
expect(result.interactionSummary.duplicateEvidenceCount).toBe(0);
expect(result.interactionSummary.conflictingBranchCount).toBe(0);
});
it("detects duplicate evidence instead of corroboration", () => {
const graph = makeParentWithBranches([
makeResolvedChild(
"n-a",
"Bank statement shows increased debtor balance",
"same-bank",
{
evidenceIds: ["same-bank"],
},
),
makeResolvedChild(
"n-b",
"Delayed receipts also cite the bank statement",
"same-bank",
{
evidenceIds: ["same-bank"],
},
),
]);
const parentNode = graph.nodes.find((node) => node.id === "n-parent");
const result = evaluateBranchInteractions({ parentNode, graph });
expect(result.interactionSummary.duplicateEvidenceCount).toBe(1);
expect(result.interactionSummary.corroboratingBranchCount).toBe(0);
});
it("detects conflicting branches", () => {
const graph = makeParentWithBranches([
makeResolvedChild("n-a", "Revenue recognised correctly", "correctly"),
makeNode({
id: "n-b",
label: "Revenue recognised incorrectly",
description: "Revenue recognised incorrectly",
kind: "unknown",
status: "contradicted",
confidence: "medium",
value: "incorrectly",
}),
]);
const parentNode = graph.nodes.find((node) => node.id === "n-parent");
const result = evaluateBranchInteractions({ parentNode, graph });
expect(result.interactionSummary.conflictingBranchCount).toBe(1);
});
});
describe("cross-branch corroboration effects", () => {
function applyToGraph(children, resolvedIds, contradictedIds = []) {
const graph = makeParentWithBranches(children);
return applyValidatedProposal({
situationGraph: graph,
proposal: {
addedNodes: [
makeNode({
id: "n-anchor",
label: "Update anchor",
description:
"Anchor state introduced by the answer because the update must contain a meaningful change.",
kind: "state",
status: "known",
confidence: "low",
}),
],
updatedNodes: [
...resolvedIds.map((id) => ({
nodeId: id,
previousStatus: "unknown",
newStatus: "resolved",
previousValue: null,
newValue: `answer:${id}`,
reason: "resolved child",
})),
...contradictedIds.map((id) => ({
nodeId: id,
previousStatus: "unknown",
newStatus: "contradicted",
previousValue: null,
newValue: `contradiction:${id}`,
reason: "contradicted child",
})),
],
addedEdges: [],
removedEdgeIds: [],
resolvedUnknownNodeIds: resolvedIds,
affectedNodeIds: [],
selectedQuestion: null,
},
previousQuestion: "What evidence would clarify this branch?",
answer: "deterministic branch update",
});
}
it("independent corroboration increases justified confidence without reaching high on incomplete parent", () => {
const result = applyToGraph(
[
makeUnknownBranch(
"n-a",
"Debtor balance increased",
"Debtor balance increased",
),
makeUnknownBranch(
"n-b",
"Cash receipts delayed",
"Cash receipts delayed",
),
makeUnknownBranch(
"n-c",
"Possible one-off event during the period",
"Possible one-off event during the period",
),
makeUnknownBranch(
"n-d",
"Whether the two observations reflect different timing",
"Whether the two observations reflect different timing",
),
],
["n-a", "n-b"],
);
expect(result.success).toBe(true);
expect(result.interactionSummary?.corroboratingBranchCount).toBeGreaterThan(
0,
);
expect(result.interactionSummary?.duplicateEvidenceCount).toBe(0);
expect(result.parentConfidenceAfter).toBe("medium");
expect(result.confidenceCapReason).toBe(
"independent_corroboration_with_incomplete_parent",
);
});
it("duplicate evidence does not increase confidence", () => {
const result = applyToGraph(
[
makeUnknownBranch(
"n-a",
"Bank statement shows increased debtor balance",
"Bank statement shows increased debtor balance",
{ evidenceIds: ["same-bank"] },
),
makeUnknownBranch(
"n-b",
"Delayed receipts also cite the bank statement",
"Delayed receipts also cite the bank statement",
{ evidenceIds: ["same-bank"] },
),
makeUnknownBranch(
"n-c",
"Possible one-off event during the period",
"Possible one-off event during the period",
),
],
["n-a", "n-b"],
);
expect(result.success).toBe(true);
expect(result.interactionSummary?.duplicateEvidenceCount).toBeGreaterThan(
0,
);
expect(result.interactionSummary?.corroboratingBranchCount).toBe(0);
expect(result.confidenceCapReason).toBe(
"duplicate_evidence_no_extra_confidence",
);
});
it("conflicting evidence caps confidence", () => {
const result = applyToGraph(
[
makeUnknownBranch(
"n-a",
"Revenue recognised correctly",
"Revenue recognised correctly",
),
makeUnknownBranch(
"n-b",
"Revenue recognised incorrectly",
"Revenue recognised incorrectly",
),
makeUnknownBranch(
"n-c",
"Possible one-off event during the period",
"Possible one-off event during the period",
),
],
["n-a"],
["n-b"],
);
expect(result.success).toBe(true);
expect(result.interactionSummary?.conflictingBranchCount).toBeGreaterThan(
0,
);
expect(result.conclusionConfidenceAfter).toBe("low");
});
it("independent branches stay interaction-neutral", () => {
const result = applyToGraph(
[
makeUnknownBranch(
"n-a",
"Marketing campaign changed traffic",
"Marketing campaign changed traffic",
),
makeUnknownBranch(
"n-b",
"Equipment maintenance occurred",
"Equipment maintenance occurred",
),
],
["n-a"],
);
expect(result.success).toBe(true);
expect(result.interactionSummary?.independentBranchCount).toBeGreaterThan(
0,
);
});
});
tests/graph/upward-propagation.test.js
+2 -2
View File
@@ -257,7 +257,7 @@ describe("upward propagation", () => {
expect(result.parentConfidenceBefore).toBe("medium");
expect(result.parentConfidenceAfter).toBe("medium");
expect(result.evidenceConfidenceBefore).toBe("medium");
expect(result.evidenceConfidenceAfter).toBe("high");
expect(result.evidenceConfidenceAfter).toBe("medium");
expect(result.completenessBefore).toBe("empty");
expect(result.completenessAfter).toBe("partial");
expect(result.conclusionConfidenceBefore).toBe("low");
@@ -287,7 +287,7 @@ describe("upward propagation", () => {
status: "provisional",
confidence: "medium",
confidenceAssessment: {
evidenceConfidence: "high",
evidenceConfidence: "medium",
completenessStatus: "partial",
conclusionConfidence: "medium",
},