feat: separate confidence from reasoning completeness
This commit is contained in:
@@ -66,6 +66,11 @@ export default function GraphUpdateView({ updateResult }) {
|
|||||||
<div className="text-xs text-gray-600">
|
<div className="text-xs text-gray-600">
|
||||||
{node.kind} · {node.confidence}
|
{node.kind} · {node.confidence}
|
||||||
</div>
|
</div>
|
||||||
|
{node.confidenceAssessment && (
|
||||||
|
<div className="text-xs text-gray-600">
|
||||||
|
evidence {node.confidenceAssessment.evidenceConfidence} · completeness {node.confidenceAssessment.completenessStatus} · conclusion {node.confidenceAssessment.conclusionConfidence}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
{(update?.previousStatus || update?.newStatus || node.status) && (
|
{(update?.previousStatus || update?.newStatus || node.status) && (
|
||||||
<div className="text-xs text-gray-700">
|
<div className="text-xs text-gray-700">
|
||||||
{update?.previousStatus ? `Previous status: ${update.previousStatus}` : null}
|
{update?.previousStatus ? `Previous status: ${update.previousStatus}` : null}
|
||||||
|
|||||||
@@ -40,6 +40,11 @@ function NodeGroup({
|
|||||||
<span className="font-medium text-gray-900">{node.label}</span>
|
<span className="font-medium text-gray-900">{node.label}</span>
|
||||||
<NodeBadge tone="blue">{node.status}</NodeBadge>
|
<NodeBadge tone="blue">{node.status}</NodeBadge>
|
||||||
<NodeBadge tone="green">{node.confidence}</NodeBadge>
|
<NodeBadge tone="green">{node.confidence}</NodeBadge>
|
||||||
|
{node.confidenceAssessment?.completenessStatus && (
|
||||||
|
<NodeBadge tone="purple">
|
||||||
|
completeness: {node.confidenceAssessment.completenessStatus}
|
||||||
|
</NodeBadge>
|
||||||
|
)}
|
||||||
{resolvedNodeIds.has(node.id) && (
|
{resolvedNodeIds.has(node.id) && (
|
||||||
<NodeBadge tone="red">resolved unknown</NodeBadge>
|
<NodeBadge tone="red">resolved unknown</NodeBadge>
|
||||||
)}
|
)}
|
||||||
@@ -59,6 +64,12 @@ function NodeGroup({
|
|||||||
{node.description && node.description !== node.label && (
|
{node.description && node.description !== node.label && (
|
||||||
<p className="mt-1 text-gray-600">{node.description}</p>
|
<p className="mt-1 text-gray-600">{node.description}</p>
|
||||||
)}
|
)}
|
||||||
|
{node.confidenceAssessment && (
|
||||||
|
<p className="mt-1 text-xs text-gray-500">
|
||||||
|
evidence: {node.confidenceAssessment.evidenceConfidence} ·
|
||||||
|
conclusion: {node.confidenceAssessment.conclusionConfidence}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
</li>
|
</li>
|
||||||
))}
|
))}
|
||||||
</ul>
|
</ul>
|
||||||
|
|||||||
@@ -119,6 +119,8 @@ rather than asking the full broad explanation node directly.
|
|||||||
|
|
||||||
Recursive reasoning is complete only when decomposition and reconstruction are both deterministic.
|
Recursive reasoning is complete only when decomposition and reconstruction are both deterministic.
|
||||||
|
|
||||||
|
Confidence must not outrun completeness or evidence.
|
||||||
|
|
||||||
For this experiment, reconstruction now behaves as follows:
|
For this experiment, reconstruction now behaves as follows:
|
||||||
|
|
||||||
- when a child unknown resolves, that child keeps its own resolved status and answer evidence
|
- when a child unknown resolves, that child keeps its own resolved status and answer evidence
|
||||||
@@ -132,10 +134,26 @@ For the current conservative completion rule:
|
|||||||
- **one resolved child** → parent becomes `provisional` with higher confidence, but remains unresolved
|
- **one resolved child** → parent becomes `provisional` with higher confidence, but remains unresolved
|
||||||
- **all direct child unknowns resolved** → parent resolves deterministically with `high` confidence
|
- **all direct child unknowns resolved** → parent resolves deterministically with `high` confidence
|
||||||
|
|
||||||
|
The confidence model is now explicitly separated into:
|
||||||
|
|
||||||
|
- **evidence confidence**: how trustworthy the currently attached support is
|
||||||
|
- **completeness**: whether the required direct child structure is empty, partial, or complete
|
||||||
|
- **conclusion confidence**: how strongly the current parent state is justified given both evidence and completeness
|
||||||
|
|
||||||
|
Deterministic propagation rules now enforce:
|
||||||
|
|
||||||
|
- one resolved child may raise evidence confidence
|
||||||
|
- unresolved direct children cap conclusion confidence
|
||||||
|
- contradictory direct children block high conclusion confidence
|
||||||
|
- duplicate evidence does not increase confidence
|
||||||
|
- status changes do not raise confidence on their own
|
||||||
|
- parent resolution still requires the separate completion rule
|
||||||
|
|
||||||
Example progression:
|
Example progression:
|
||||||
|
|
||||||
- parent before: `unknown`, `medium`
|
- parent before: `unknown`, `medium`
|
||||||
- after resolving `How the two observations were measured`: parent becomes `provisional`, `high`
|
- after resolving `How the two observations were measured`: parent becomes `provisional`, `medium`
|
||||||
|
- evidence confidence becomes `high`, completeness becomes `partial`, conclusion confidence becomes `medium`
|
||||||
- next sibling becomes selectable and the engine moves on without recreating the resolved child
|
- next sibling becomes selectable and the engine moves on without recreating the resolved child
|
||||||
|
|
||||||
## Interpretation
|
## Interpretation
|
||||||
|
|||||||
+170
-14
@@ -439,6 +439,30 @@ function appendUniqueValue(values = [], nextValue) {
|
|||||||
: values;
|
: values;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getNodeConfidenceAssessment(node) {
|
||||||
|
return (
|
||||||
|
node?.confidenceAssessment || {
|
||||||
|
evidenceConfidence: node?.confidence ?? "medium",
|
||||||
|
completenessStatus:
|
||||||
|
node?.status === "resolved"
|
||||||
|
? "complete"
|
||||||
|
: node?.status === "provisional"
|
||||||
|
? "partial"
|
||||||
|
: "empty",
|
||||||
|
conclusionConfidence:
|
||||||
|
node?.status === "resolved"
|
||||||
|
? (node?.confidence ?? "high")
|
||||||
|
: node?.status === "provisional"
|
||||||
|
? (node?.confidence ?? "medium")
|
||||||
|
: "low",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function confidenceFromAssessment(assessment) {
|
||||||
|
return assessment?.conclusionConfidence ?? "medium";
|
||||||
|
}
|
||||||
|
|
||||||
function upsertProposalNodeUpdate(proposalSnapshot, update) {
|
function upsertProposalNodeUpdate(proposalSnapshot, update) {
|
||||||
const existing = proposalSnapshot.updatedNodes.find(
|
const existing = proposalSnapshot.updatedNodes.find(
|
||||||
(candidate) => candidate.nodeId === update.nodeId,
|
(candidate) => candidate.nodeId === update.nodeId,
|
||||||
@@ -521,16 +545,6 @@ function buildAncestorChain(graph, node) {
|
|||||||
queue.push(candidate.id);
|
queue.push(candidate.id);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const edge of graph.edges || []) {
|
|
||||||
if (
|
|
||||||
edge.fromNodeId !== parentNode.id &&
|
|
||||||
edge.toNodeId === parentNode.id &&
|
|
||||||
edge.relationship === "depends_on"
|
|
||||||
) {
|
|
||||||
queue.push(edge.fromNodeId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return chain;
|
return chain;
|
||||||
@@ -562,28 +576,83 @@ function computeParentProgressState(graph, parentNode) {
|
|||||||
const progressedChildren = childUnknowns.filter((child) =>
|
const progressedChildren = childUnknowns.filter((child) =>
|
||||||
["resolved", "provisional"].includes(child.status),
|
["resolved", "provisional"].includes(child.status),
|
||||||
);
|
);
|
||||||
|
const contradictoryChildren = childUnknowns.filter(
|
||||||
|
(child) => child.status === "contradicted",
|
||||||
|
);
|
||||||
const totalChildren = childUnknowns.length;
|
const totalChildren = childUnknowns.length;
|
||||||
|
const resolvedCount = resolvedChildren.length;
|
||||||
|
const unresolvedCount = childUnknowns.filter(
|
||||||
|
(child) => child.status !== "resolved",
|
||||||
|
).length;
|
||||||
|
const beforeAssessment = getNodeConfidenceAssessment(parentNode);
|
||||||
|
|
||||||
if (totalChildren === 0) {
|
if (totalChildren === 0) {
|
||||||
|
const nextAssessment = {
|
||||||
|
evidenceConfidence: beforeAssessment.evidenceConfidence,
|
||||||
|
completenessStatus: beforeAssessment.completenessStatus,
|
||||||
|
conclusionConfidence: beforeAssessment.conclusionConfidence,
|
||||||
|
};
|
||||||
return {
|
return {
|
||||||
totalChildren,
|
totalChildren,
|
||||||
resolvedChildren,
|
resolvedChildren,
|
||||||
progressedChildren,
|
progressedChildren,
|
||||||
|
contradictoryChildren,
|
||||||
nextStatus: parentNode.status,
|
nextStatus: parentNode.status,
|
||||||
nextConfidence: parentNode.confidence,
|
nextConfidence: confidenceFromAssessment(nextAssessment),
|
||||||
|
nextConfidenceAssessment: nextAssessment,
|
||||||
parentResolved: parentNode.status === "resolved",
|
parentResolved: parentNode.status === "resolved",
|
||||||
|
confidenceCapReason: "no_child_structure",
|
||||||
reason: "Parent has no child unknowns to aggregate.",
|
reason: "Parent has no child unknowns to aggregate.",
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let nextAssessment;
|
||||||
|
let confidenceCapReason;
|
||||||
|
|
||||||
|
if (contradictoryChildren.length > 0) {
|
||||||
|
nextAssessment = {
|
||||||
|
evidenceConfidence: resolvedCount > 0 ? "medium" : "low",
|
||||||
|
completenessStatus: resolvedCount === 0 ? "empty" : "partial",
|
||||||
|
conclusionConfidence: "low",
|
||||||
|
};
|
||||||
|
confidenceCapReason = "contradictory_direct_children";
|
||||||
|
} else if (resolvedCount === 0) {
|
||||||
|
nextAssessment = {
|
||||||
|
evidenceConfidence: "low",
|
||||||
|
completenessStatus: "empty",
|
||||||
|
conclusionConfidence: "low",
|
||||||
|
};
|
||||||
|
confidenceCapReason = "no_resolved_direct_children";
|
||||||
|
} else if (resolvedCount < totalChildren) {
|
||||||
|
nextAssessment = {
|
||||||
|
evidenceConfidence: "high",
|
||||||
|
completenessStatus: "partial",
|
||||||
|
conclusionConfidence: "medium",
|
||||||
|
};
|
||||||
|
confidenceCapReason = "unresolved_direct_children_cap_conclusion";
|
||||||
|
} else {
|
||||||
|
nextAssessment = {
|
||||||
|
evidenceConfidence: "high",
|
||||||
|
completenessStatus: "complete",
|
||||||
|
conclusionConfidence: "high",
|
||||||
|
};
|
||||||
|
confidenceCapReason = null;
|
||||||
|
}
|
||||||
|
|
||||||
if (resolvedChildren.length === totalChildren) {
|
if (resolvedChildren.length === totalChildren) {
|
||||||
return {
|
return {
|
||||||
totalChildren,
|
totalChildren,
|
||||||
resolvedChildren,
|
resolvedChildren,
|
||||||
progressedChildren,
|
progressedChildren,
|
||||||
|
contradictoryChildren,
|
||||||
nextStatus: "resolved",
|
nextStatus: "resolved",
|
||||||
nextConfidence: "high",
|
nextConfidence: confidenceFromAssessment(nextAssessment),
|
||||||
|
nextConfidenceAssessment: nextAssessment,
|
||||||
parentResolved: true,
|
parentResolved: true,
|
||||||
|
resolvedDirectChildren: resolvedCount,
|
||||||
|
unresolvedDirectChildren: unresolvedCount,
|
||||||
|
contradictoryDirectChildren: contradictoryChildren.length,
|
||||||
|
confidenceCapReason,
|
||||||
reason:
|
reason:
|
||||||
"All direct child unknowns are resolved, so the parent can now resolve deterministically.",
|
"All direct child unknowns are resolved, so the parent can now resolve deterministically.",
|
||||||
};
|
};
|
||||||
@@ -594,9 +663,15 @@ function computeParentProgressState(graph, parentNode) {
|
|||||||
totalChildren,
|
totalChildren,
|
||||||
resolvedChildren,
|
resolvedChildren,
|
||||||
progressedChildren,
|
progressedChildren,
|
||||||
|
contradictoryChildren,
|
||||||
nextStatus: "provisional",
|
nextStatus: "provisional",
|
||||||
nextConfidence: "high",
|
nextConfidence: confidenceFromAssessment(nextAssessment),
|
||||||
|
nextConfidenceAssessment: nextAssessment,
|
||||||
parentResolved: false,
|
parentResolved: false,
|
||||||
|
resolvedDirectChildren: resolvedCount,
|
||||||
|
unresolvedDirectChildren: unresolvedCount,
|
||||||
|
contradictoryDirectChildren: contradictoryChildren.length,
|
||||||
|
confidenceCapReason,
|
||||||
reason:
|
reason:
|
||||||
"At least one direct child has been progressed, so the parent becomes provisional but remains unresolved until all direct children are resolved.",
|
"At least one direct child has been progressed, so the parent becomes provisional but remains unresolved until all direct children are resolved.",
|
||||||
};
|
};
|
||||||
@@ -606,9 +681,15 @@ function computeParentProgressState(graph, parentNode) {
|
|||||||
totalChildren,
|
totalChildren,
|
||||||
resolvedChildren,
|
resolvedChildren,
|
||||||
progressedChildren,
|
progressedChildren,
|
||||||
|
contradictoryChildren,
|
||||||
nextStatus: parentNode.status,
|
nextStatus: parentNode.status,
|
||||||
nextConfidence: parentNode.confidence,
|
nextConfidence: confidenceFromAssessment(nextAssessment),
|
||||||
|
nextConfidenceAssessment: nextAssessment,
|
||||||
parentResolved: parentNode.status === "resolved",
|
parentResolved: parentNode.status === "resolved",
|
||||||
|
resolvedDirectChildren: resolvedCount,
|
||||||
|
unresolvedDirectChildren: unresolvedCount,
|
||||||
|
contradictoryDirectChildren: contradictoryChildren.length,
|
||||||
|
confidenceCapReason,
|
||||||
reason: "No direct child progress exists yet for the parent.",
|
reason: "No direct child progress exists yet for the parent.",
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -635,6 +716,17 @@ export function propagateResolvedChildEvidence({
|
|||||||
parentStatusAfter: null,
|
parentStatusAfter: null,
|
||||||
parentConfidenceBefore: null,
|
parentConfidenceBefore: null,
|
||||||
parentConfidenceAfter: null,
|
parentConfidenceAfter: null,
|
||||||
|
evidenceConfidenceBefore: null,
|
||||||
|
evidenceConfidenceAfter: null,
|
||||||
|
completenessBefore: null,
|
||||||
|
completenessAfter: null,
|
||||||
|
conclusionConfidenceBefore: null,
|
||||||
|
conclusionConfidenceAfter: null,
|
||||||
|
resolvedDirectChildren: 0,
|
||||||
|
unresolvedDirectChildren: 0,
|
||||||
|
contradictoryDirectChildren: 0,
|
||||||
|
confidenceCapReason: null,
|
||||||
|
ancestorPropagationStoppedReason: "no_resolved_child_propagation_needed",
|
||||||
affectedAncestorIds: [],
|
affectedAncestorIds: [],
|
||||||
nextSelectedSibling: null,
|
nextSelectedSibling: null,
|
||||||
parentResolved: false,
|
parentResolved: false,
|
||||||
@@ -646,6 +738,7 @@ export function propagateResolvedChildEvidence({
|
|||||||
syncParentChildReferences(graph);
|
syncParentChildReferences(graph);
|
||||||
const propagationEvents = [];
|
const propagationEvents = [];
|
||||||
const affectedAncestorIds = new Set();
|
const affectedAncestorIds = new Set();
|
||||||
|
let ancestorPropagationStoppedReason = "no_ancestor_state_changed";
|
||||||
|
|
||||||
for (const resolvedChildNode of resolvedChildNodes) {
|
for (const resolvedChildNode of resolvedChildNodes) {
|
||||||
const liveChildNode = graph.nodes.find(
|
const liveChildNode = graph.nodes.find(
|
||||||
@@ -662,10 +755,24 @@ export function propagateResolvedChildEvidence({
|
|||||||
for (const ancestorNode of ancestorChain) {
|
for (const ancestorNode of ancestorChain) {
|
||||||
const beforeStatus = ancestorNode.status;
|
const beforeStatus = ancestorNode.status;
|
||||||
const beforeConfidence = ancestorNode.confidence;
|
const beforeConfidence = ancestorNode.confidence;
|
||||||
|
const beforeAssessment = getNodeConfidenceAssessment(ancestorNode);
|
||||||
const progressState = computeParentProgressState(graph, ancestorNode);
|
const progressState = computeParentProgressState(graph, ancestorNode);
|
||||||
|
|
||||||
ancestorNode.status = progressState.nextStatus;
|
ancestorNode.status = progressState.nextStatus;
|
||||||
ancestorNode.confidence = progressState.nextConfidence;
|
ancestorNode.confidence = progressState.nextConfidence;
|
||||||
|
ancestorNode.confidenceAssessment =
|
||||||
|
progressState.nextConfidenceAssessment;
|
||||||
|
|
||||||
|
if (
|
||||||
|
beforeStatus === progressState.nextStatus &&
|
||||||
|
beforeConfidence === progressState.nextConfidence &&
|
||||||
|
JSON.stringify(beforeAssessment) ===
|
||||||
|
JSON.stringify(progressState.nextConfidenceAssessment)
|
||||||
|
) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
ancestorPropagationStoppedReason = "ancestor_state_changed";
|
||||||
|
|
||||||
if (progressState.parentResolved) {
|
if (progressState.parentResolved) {
|
||||||
ensureResolvedUnknownId(proposalSnapshot, ancestorNode.id);
|
ensureResolvedUnknownId(proposalSnapshot, ancestorNode.id);
|
||||||
@@ -688,6 +795,19 @@ export function propagateResolvedChildEvidence({
|
|||||||
parentStatusAfter: progressState.nextStatus,
|
parentStatusAfter: progressState.nextStatus,
|
||||||
parentConfidenceBefore: beforeConfidence,
|
parentConfidenceBefore: beforeConfidence,
|
||||||
parentConfidenceAfter: progressState.nextConfidence,
|
parentConfidenceAfter: progressState.nextConfidence,
|
||||||
|
evidenceConfidenceBefore: beforeAssessment.evidenceConfidence,
|
||||||
|
evidenceConfidenceAfter:
|
||||||
|
progressState.nextConfidenceAssessment.evidenceConfidence,
|
||||||
|
completenessBefore: beforeAssessment.completenessStatus,
|
||||||
|
completenessAfter:
|
||||||
|
progressState.nextConfidenceAssessment.completenessStatus,
|
||||||
|
conclusionConfidenceBefore: beforeAssessment.conclusionConfidence,
|
||||||
|
conclusionConfidenceAfter:
|
||||||
|
progressState.nextConfidenceAssessment.conclusionConfidence,
|
||||||
|
resolvedDirectChildren: progressState.resolvedDirectChildren,
|
||||||
|
unresolvedDirectChildren: progressState.unresolvedDirectChildren,
|
||||||
|
contradictoryDirectChildren: progressState.contradictoryDirectChildren,
|
||||||
|
confidenceCapReason: progressState.confidenceCapReason,
|
||||||
parentResolved: progressState.parentResolved,
|
parentResolved: progressState.parentResolved,
|
||||||
reason: progressState.reason,
|
reason: progressState.reason,
|
||||||
});
|
});
|
||||||
@@ -717,6 +837,17 @@ export function propagateResolvedChildEvidence({
|
|||||||
parentStatusAfter: firstEvent?.parentStatusAfter ?? null,
|
parentStatusAfter: firstEvent?.parentStatusAfter ?? null,
|
||||||
parentConfidenceBefore: firstEvent?.parentConfidenceBefore ?? null,
|
parentConfidenceBefore: firstEvent?.parentConfidenceBefore ?? null,
|
||||||
parentConfidenceAfter: firstEvent?.parentConfidenceAfter ?? null,
|
parentConfidenceAfter: firstEvent?.parentConfidenceAfter ?? null,
|
||||||
|
evidenceConfidenceBefore: firstEvent?.evidenceConfidenceBefore ?? null,
|
||||||
|
evidenceConfidenceAfter: firstEvent?.evidenceConfidenceAfter ?? null,
|
||||||
|
completenessBefore: firstEvent?.completenessBefore ?? null,
|
||||||
|
completenessAfter: firstEvent?.completenessAfter ?? null,
|
||||||
|
conclusionConfidenceBefore: firstEvent?.conclusionConfidenceBefore ?? null,
|
||||||
|
conclusionConfidenceAfter: firstEvent?.conclusionConfidenceAfter ?? null,
|
||||||
|
resolvedDirectChildren: firstEvent?.resolvedDirectChildren ?? 0,
|
||||||
|
unresolvedDirectChildren: firstEvent?.unresolvedDirectChildren ?? 0,
|
||||||
|
contradictoryDirectChildren: firstEvent?.contradictoryDirectChildren ?? 0,
|
||||||
|
confidenceCapReason: firstEvent?.confidenceCapReason ?? null,
|
||||||
|
ancestorPropagationStoppedReason,
|
||||||
affectedAncestorIds: [...affectedAncestorIds],
|
affectedAncestorIds: [...affectedAncestorIds],
|
||||||
nextSelectedSibling:
|
nextSelectedSibling:
|
||||||
siblingSelection?.status === "selected" ? siblingSelection.nodeId : null,
|
siblingSelection?.status === "selected" ? siblingSelection.nodeId : null,
|
||||||
@@ -1767,6 +1898,20 @@ export function applyValidatedProposal({
|
|||||||
const parentStatusAfter = propagationResult.parentStatusAfter;
|
const parentStatusAfter = propagationResult.parentStatusAfter;
|
||||||
const parentConfidenceBefore = propagationResult.parentConfidenceBefore;
|
const parentConfidenceBefore = propagationResult.parentConfidenceBefore;
|
||||||
const parentConfidenceAfter = propagationResult.parentConfidenceAfter;
|
const parentConfidenceAfter = propagationResult.parentConfidenceAfter;
|
||||||
|
const evidenceConfidenceBefore = propagationResult.evidenceConfidenceBefore;
|
||||||
|
const evidenceConfidenceAfter = propagationResult.evidenceConfidenceAfter;
|
||||||
|
const completenessBefore = propagationResult.completenessBefore;
|
||||||
|
const completenessAfter = propagationResult.completenessAfter;
|
||||||
|
const conclusionConfidenceBefore =
|
||||||
|
propagationResult.conclusionConfidenceBefore;
|
||||||
|
const conclusionConfidenceAfter = propagationResult.conclusionConfidenceAfter;
|
||||||
|
const resolvedDirectChildren = propagationResult.resolvedDirectChildren;
|
||||||
|
const unresolvedDirectChildren = propagationResult.unresolvedDirectChildren;
|
||||||
|
const contradictoryDirectChildren =
|
||||||
|
propagationResult.contradictoryDirectChildren;
|
||||||
|
const confidenceCapReason = propagationResult.confidenceCapReason;
|
||||||
|
const ancestorPropagationStoppedReason =
|
||||||
|
propagationResult.ancestorPropagationStoppedReason;
|
||||||
const affectedAncestorIds = propagationResult.affectedAncestorIds;
|
const affectedAncestorIds = propagationResult.affectedAncestorIds;
|
||||||
const nextSelectedSibling = propagationResult.nextSelectedSibling;
|
const nextSelectedSibling = propagationResult.nextSelectedSibling;
|
||||||
const parentResolved = propagationResult.parentResolved;
|
const parentResolved = propagationResult.parentResolved;
|
||||||
@@ -1894,6 +2039,17 @@ export function applyValidatedProposal({
|
|||||||
parentStatusAfter,
|
parentStatusAfter,
|
||||||
parentConfidenceBefore,
|
parentConfidenceBefore,
|
||||||
parentConfidenceAfter,
|
parentConfidenceAfter,
|
||||||
|
evidenceConfidenceBefore,
|
||||||
|
evidenceConfidenceAfter,
|
||||||
|
completenessBefore,
|
||||||
|
completenessAfter,
|
||||||
|
conclusionConfidenceBefore,
|
||||||
|
conclusionConfidenceAfter,
|
||||||
|
resolvedDirectChildren,
|
||||||
|
unresolvedDirectChildren,
|
||||||
|
contradictoryDirectChildren,
|
||||||
|
confidenceCapReason,
|
||||||
|
ancestorPropagationStoppedReason,
|
||||||
affectedAncestorIds,
|
affectedAncestorIds,
|
||||||
nextSelectedSibling,
|
nextSelectedSibling,
|
||||||
parentResolved,
|
parentResolved,
|
||||||
|
|||||||
@@ -110,6 +110,17 @@ function buildUpdateDiagnostics({
|
|||||||
parentStatusAfter,
|
parentStatusAfter,
|
||||||
parentConfidenceBefore,
|
parentConfidenceBefore,
|
||||||
parentConfidenceAfter,
|
parentConfidenceAfter,
|
||||||
|
evidenceConfidenceBefore,
|
||||||
|
evidenceConfidenceAfter,
|
||||||
|
completenessBefore,
|
||||||
|
completenessAfter,
|
||||||
|
conclusionConfidenceBefore,
|
||||||
|
conclusionConfidenceAfter,
|
||||||
|
resolvedDirectChildren,
|
||||||
|
unresolvedDirectChildren,
|
||||||
|
contradictoryDirectChildren,
|
||||||
|
confidenceCapReason,
|
||||||
|
ancestorPropagationStoppedReason,
|
||||||
affectedAncestorIds,
|
affectedAncestorIds,
|
||||||
nextSelectedSibling,
|
nextSelectedSibling,
|
||||||
parentResolved,
|
parentResolved,
|
||||||
@@ -163,6 +174,17 @@ function buildUpdateDiagnostics({
|
|||||||
parentStatusAfter: parentStatusAfter ?? null,
|
parentStatusAfter: parentStatusAfter ?? null,
|
||||||
parentConfidenceBefore: parentConfidenceBefore ?? null,
|
parentConfidenceBefore: parentConfidenceBefore ?? null,
|
||||||
parentConfidenceAfter: parentConfidenceAfter ?? null,
|
parentConfidenceAfter: parentConfidenceAfter ?? null,
|
||||||
|
evidenceConfidenceBefore: evidenceConfidenceBefore ?? null,
|
||||||
|
evidenceConfidenceAfter: evidenceConfidenceAfter ?? null,
|
||||||
|
completenessBefore: completenessBefore ?? null,
|
||||||
|
completenessAfter: completenessAfter ?? null,
|
||||||
|
conclusionConfidenceBefore: conclusionConfidenceBefore ?? null,
|
||||||
|
conclusionConfidenceAfter: conclusionConfidenceAfter ?? null,
|
||||||
|
resolvedDirectChildren: resolvedDirectChildren ?? 0,
|
||||||
|
unresolvedDirectChildren: unresolvedDirectChildren ?? 0,
|
||||||
|
contradictoryDirectChildren: contradictoryDirectChildren ?? 0,
|
||||||
|
confidenceCapReason: confidenceCapReason ?? null,
|
||||||
|
ancestorPropagationStoppedReason: ancestorPropagationStoppedReason ?? null,
|
||||||
affectedAncestorIds: affectedAncestorIds ?? [],
|
affectedAncestorIds: affectedAncestorIds ?? [],
|
||||||
nextSelectedSibling: nextSelectedSibling ?? null,
|
nextSelectedSibling: nextSelectedSibling ?? null,
|
||||||
parentResolved: parentResolved ?? false,
|
parentResolved: parentResolved ?? false,
|
||||||
@@ -440,6 +462,17 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
|
|||||||
parentStatusAfter: null,
|
parentStatusAfter: null,
|
||||||
parentConfidenceBefore: null,
|
parentConfidenceBefore: null,
|
||||||
parentConfidenceAfter: null,
|
parentConfidenceAfter: null,
|
||||||
|
evidenceConfidenceBefore: null,
|
||||||
|
evidenceConfidenceAfter: null,
|
||||||
|
completenessBefore: null,
|
||||||
|
completenessAfter: null,
|
||||||
|
conclusionConfidenceBefore: null,
|
||||||
|
conclusionConfidenceAfter: null,
|
||||||
|
resolvedDirectChildren: 0,
|
||||||
|
unresolvedDirectChildren: 0,
|
||||||
|
contradictoryDirectChildren: 0,
|
||||||
|
confidenceCapReason: null,
|
||||||
|
ancestorPropagationStoppedReason: null,
|
||||||
affectedAncestorIds: [],
|
affectedAncestorIds: [],
|
||||||
nextSelectedSibling: null,
|
nextSelectedSibling: null,
|
||||||
parentResolved: false,
|
parentResolved: false,
|
||||||
@@ -508,6 +541,20 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
|
|||||||
parentStatusAfter: applicationResult.parentStatusAfter,
|
parentStatusAfter: applicationResult.parentStatusAfter,
|
||||||
parentConfidenceBefore: applicationResult.parentConfidenceBefore,
|
parentConfidenceBefore: applicationResult.parentConfidenceBefore,
|
||||||
parentConfidenceAfter: applicationResult.parentConfidenceAfter,
|
parentConfidenceAfter: applicationResult.parentConfidenceAfter,
|
||||||
|
evidenceConfidenceBefore: applicationResult.evidenceConfidenceBefore,
|
||||||
|
evidenceConfidenceAfter: applicationResult.evidenceConfidenceAfter,
|
||||||
|
completenessBefore: applicationResult.completenessBefore,
|
||||||
|
completenessAfter: applicationResult.completenessAfter,
|
||||||
|
conclusionConfidenceBefore:
|
||||||
|
applicationResult.conclusionConfidenceBefore,
|
||||||
|
conclusionConfidenceAfter: applicationResult.conclusionConfidenceAfter,
|
||||||
|
resolvedDirectChildren: applicationResult.resolvedDirectChildren,
|
||||||
|
unresolvedDirectChildren: applicationResult.unresolvedDirectChildren,
|
||||||
|
contradictoryDirectChildren:
|
||||||
|
applicationResult.contradictoryDirectChildren,
|
||||||
|
confidenceCapReason: applicationResult.confidenceCapReason,
|
||||||
|
ancestorPropagationStoppedReason:
|
||||||
|
applicationResult.ancestorPropagationStoppedReason,
|
||||||
affectedAncestorIds: applicationResult.affectedAncestorIds,
|
affectedAncestorIds: applicationResult.affectedAncestorIds,
|
||||||
nextSelectedSibling: applicationResult.nextSelectedSibling,
|
nextSelectedSibling: applicationResult.nextSelectedSibling,
|
||||||
parentResolved: applicationResult.parentResolved,
|
parentResolved: applicationResult.parentResolved,
|
||||||
@@ -560,6 +607,17 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
|
|||||||
parentStatusAfter: null,
|
parentStatusAfter: null,
|
||||||
parentConfidenceBefore: null,
|
parentConfidenceBefore: null,
|
||||||
parentConfidenceAfter: null,
|
parentConfidenceAfter: null,
|
||||||
|
evidenceConfidenceBefore: null,
|
||||||
|
evidenceConfidenceAfter: null,
|
||||||
|
completenessBefore: null,
|
||||||
|
completenessAfter: null,
|
||||||
|
conclusionConfidenceBefore: null,
|
||||||
|
conclusionConfidenceAfter: null,
|
||||||
|
resolvedDirectChildren: 0,
|
||||||
|
unresolvedDirectChildren: 0,
|
||||||
|
contradictoryDirectChildren: 0,
|
||||||
|
confidenceCapReason: null,
|
||||||
|
ancestorPropagationStoppedReason: null,
|
||||||
affectedAncestorIds: [],
|
affectedAncestorIds: [],
|
||||||
nextSelectedSibling: null,
|
nextSelectedSibling: null,
|
||||||
parentResolved: false,
|
parentResolved: false,
|
||||||
|
|||||||
@@ -36,6 +36,20 @@ export const ConfidenceLevel = /** @type {const} */ ({
|
|||||||
high: "high",
|
high: "high",
|
||||||
});
|
});
|
||||||
|
|
||||||
|
export const CompletenessStatus = /** @type {const} */ ({
|
||||||
|
empty: "empty",
|
||||||
|
partial: "partial",
|
||||||
|
complete: "complete",
|
||||||
|
});
|
||||||
|
|
||||||
|
export const confidenceAssessmentSchema = z
|
||||||
|
.object({
|
||||||
|
evidenceConfidence: z.enum(Object.values(ConfidenceLevel)),
|
||||||
|
completenessStatus: z.enum(Object.values(CompletenessStatus)),
|
||||||
|
conclusionConfidence: z.enum(Object.values(ConfidenceLevel)),
|
||||||
|
})
|
||||||
|
.strict();
|
||||||
|
|
||||||
// ── SituationNode ────────────────────────────────────
|
// ── SituationNode ────────────────────────────────────
|
||||||
|
|
||||||
export const situationNodeSchema = z.object({
|
export const situationNodeSchema = z.object({
|
||||||
@@ -45,6 +59,7 @@ export const situationNodeSchema = z.object({
|
|||||||
kind: z.enum(Object.values(SituationKind)),
|
kind: z.enum(Object.values(SituationKind)),
|
||||||
status: z.enum(Object.values(SituationStatus)),
|
status: z.enum(Object.values(SituationStatus)),
|
||||||
confidence: z.enum(Object.values(ConfidenceLevel)),
|
confidence: z.enum(Object.values(ConfidenceLevel)),
|
||||||
|
confidenceAssessment: confidenceAssessmentSchema.optional(),
|
||||||
value: z.union([z.string(), z.number(), z.null()]).nullable().optional(),
|
value: z.union([z.string(), z.number(), z.null()]).nullable().optional(),
|
||||||
unit: z.string().nullable().optional(),
|
unit: z.string().nullable().optional(),
|
||||||
evidenceIds: z.array(z.string()).default([]),
|
evidenceIds: z.array(z.string()).default([]),
|
||||||
@@ -188,6 +203,7 @@ export function makeNode(opts) {
|
|||||||
kind: opts.kind ?? "observation",
|
kind: opts.kind ?? "observation",
|
||||||
status: opts.status ?? "unknown",
|
status: opts.status ?? "unknown",
|
||||||
confidence: opts.confidence ?? "medium",
|
confidence: opts.confidence ?? "medium",
|
||||||
|
confidenceAssessment: opts.confidenceAssessment,
|
||||||
value: opts.value ?? null,
|
value: opts.value ?? null,
|
||||||
unit: opts.unit ?? null,
|
unit: opts.unit ?? null,
|
||||||
evidenceIds: opts.evidenceIds ?? [],
|
evidenceIds: opts.evidenceIds ?? [],
|
||||||
|
|||||||
@@ -0,0 +1,179 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { applyValidatedProposal } from "@/lib/graph/apply-proposal.js";
|
||||||
|
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||||
|
|
||||||
|
function makeFixture() {
|
||||||
|
const parent = makeNode({
|
||||||
|
id: "n-parent",
|
||||||
|
label: "Explanation for why revenue increased while cash fell",
|
||||||
|
description:
|
||||||
|
"Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
});
|
||||||
|
const children = [
|
||||||
|
makeNode({
|
||||||
|
id: "n-child-1",
|
||||||
|
label: "How the two observations were measured",
|
||||||
|
description: "Need evidence about the measure used for each observation.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
parentId: parent.id,
|
||||||
|
}),
|
||||||
|
makeNode({
|
||||||
|
id: "n-child-2",
|
||||||
|
label: "Whether the two observations reflect different timing",
|
||||||
|
description:
|
||||||
|
"Need to know whether the two observations reflect different timing.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
parentId: parent.id,
|
||||||
|
}),
|
||||||
|
makeNode({
|
||||||
|
id: "n-child-3",
|
||||||
|
label: "Possible change mainly affecting revenue",
|
||||||
|
description:
|
||||||
|
"Need to know whether a possible change mainly affected revenue.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
parentId: parent.id,
|
||||||
|
}),
|
||||||
|
makeNode({
|
||||||
|
id: "n-child-4",
|
||||||
|
label: "Possible one-off event during the period",
|
||||||
|
description:
|
||||||
|
"Need to know whether a possible one-off event happened during the period.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
parentId: parent.id,
|
||||||
|
}),
|
||||||
|
];
|
||||||
|
|
||||||
|
return makeGraph({
|
||||||
|
centralStatement: "Revenue increased while cash fell.",
|
||||||
|
nodes: [parent, ...children],
|
||||||
|
edges: children.map((child, index) =>
|
||||||
|
makeEdge({
|
||||||
|
id: `e-${index + 1}`,
|
||||||
|
fromNodeId: child.id,
|
||||||
|
toNodeId: parent.id,
|
||||||
|
relationship: "depends_on",
|
||||||
|
description: `${child.label} feeds the parent.`,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
activeUnknownNodeId: "n-child-1",
|
||||||
|
resolvedNodeIds: [],
|
||||||
|
currentSummary: "confidence propagation fixture",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeProposal({ resolvedIds, contradictedIds = [] }) {
|
||||||
|
return {
|
||||||
|
addedNodes: [
|
||||||
|
makeNode({
|
||||||
|
id: "n-anchor",
|
||||||
|
label: "Update anchor",
|
||||||
|
description:
|
||||||
|
"Anchor state introduced by the answer because the update must contain a meaningful change.",
|
||||||
|
kind: "state",
|
||||||
|
status: "known",
|
||||||
|
confidence: "low",
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
updatedNodes: [
|
||||||
|
...resolvedIds.map((id) => ({
|
||||||
|
nodeId: id,
|
||||||
|
previousStatus: "unknown",
|
||||||
|
newStatus: "resolved",
|
||||||
|
previousValue: null,
|
||||||
|
newValue: `answer:${id}`,
|
||||||
|
reason: "resolved child",
|
||||||
|
})),
|
||||||
|
...contradictedIds.map((id) => ({
|
||||||
|
nodeId: id,
|
||||||
|
previousStatus: "unknown",
|
||||||
|
newStatus: "contradicted",
|
||||||
|
previousValue: null,
|
||||||
|
newValue: `contradiction:${id}`,
|
||||||
|
reason: "contradictory child evidence",
|
||||||
|
})),
|
||||||
|
],
|
||||||
|
addedEdges: [],
|
||||||
|
removedEdgeIds: [],
|
||||||
|
resolvedUnknownNodeIds: resolvedIds,
|
||||||
|
affectedNodeIds: [],
|
||||||
|
selectedQuestion: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("confidence propagation", () => {
|
||||||
|
it("one of four children resolved does not yield high conclusion confidence", () => {
|
||||||
|
const result = applyValidatedProposal({
|
||||||
|
situationGraph: makeFixture(),
|
||||||
|
proposal: makeProposal({ resolvedIds: ["n-child-1"] }),
|
||||||
|
previousQuestion:
|
||||||
|
"What evidence would clarify how the two observations were measured?",
|
||||||
|
answer: "Same accounting period and same management accounts.",
|
||||||
|
});
|
||||||
|
|
||||||
|
const parent = result.updatedSituationGraph.nodes.find(
|
||||||
|
(n) => n.id === "n-parent",
|
||||||
|
);
|
||||||
|
expect(parent.status).toBe("provisional");
|
||||||
|
expect(parent.confidence).toBe("medium");
|
||||||
|
expect(parent.confidenceAssessment).toEqual({
|
||||||
|
evidenceConfidence: "high",
|
||||||
|
completenessStatus: "partial",
|
||||||
|
conclusionConfidence: "medium",
|
||||||
|
});
|
||||||
|
expect(result.confidenceCapReason).toBe(
|
||||||
|
"unresolved_direct_children_cap_conclusion",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("all children resolved with coherent evidence may yield high confidence", () => {
|
||||||
|
const result = applyValidatedProposal({
|
||||||
|
situationGraph: makeFixture(),
|
||||||
|
proposal: makeProposal({
|
||||||
|
resolvedIds: ["n-child-1", "n-child-2", "n-child-3", "n-child-4"],
|
||||||
|
}),
|
||||||
|
previousQuestion:
|
||||||
|
"What evidence would clarify how the two observations were measured?",
|
||||||
|
answer: "All direct child questions are answered.",
|
||||||
|
});
|
||||||
|
|
||||||
|
const parent = result.updatedSituationGraph.nodes.find(
|
||||||
|
(n) => n.id === "n-parent",
|
||||||
|
);
|
||||||
|
expect(parent.status).toBe("resolved");
|
||||||
|
expect(parent.confidenceAssessment).toEqual({
|
||||||
|
evidenceConfidence: "high",
|
||||||
|
completenessStatus: "complete",
|
||||||
|
conclusionConfidence: "high",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("contradictory child evidence prevents high confidence", () => {
|
||||||
|
const result = applyValidatedProposal({
|
||||||
|
situationGraph: makeFixture(),
|
||||||
|
proposal: makeProposal({
|
||||||
|
resolvedIds: ["n-child-1"],
|
||||||
|
contradictedIds: ["n-child-2"],
|
||||||
|
}),
|
||||||
|
previousQuestion:
|
||||||
|
"What evidence would clarify how the two observations were measured?",
|
||||||
|
answer: "One child resolved, another contradicted.",
|
||||||
|
});
|
||||||
|
|
||||||
|
const parent = result.updatedSituationGraph.nodes.find(
|
||||||
|
(n) => n.id === "n-parent",
|
||||||
|
);
|
||||||
|
expect(parent.confidenceAssessment.conclusionConfidence).toBe("low");
|
||||||
|
expect(result.confidenceCapReason).toBe("contradictory_direct_children");
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -255,7 +255,16 @@ describe("upward propagation", () => {
|
|||||||
expect(result.parentStatusBefore).toBe("unknown");
|
expect(result.parentStatusBefore).toBe("unknown");
|
||||||
expect(result.parentStatusAfter).toBe("provisional");
|
expect(result.parentStatusAfter).toBe("provisional");
|
||||||
expect(result.parentConfidenceBefore).toBe("medium");
|
expect(result.parentConfidenceBefore).toBe("medium");
|
||||||
expect(result.parentConfidenceAfter).toBe("high");
|
expect(result.parentConfidenceAfter).toBe("medium");
|
||||||
|
expect(result.evidenceConfidenceBefore).toBe("medium");
|
||||||
|
expect(result.evidenceConfidenceAfter).toBe("high");
|
||||||
|
expect(result.completenessBefore).toBe("empty");
|
||||||
|
expect(result.completenessAfter).toBe("partial");
|
||||||
|
expect(result.conclusionConfidenceBefore).toBe("low");
|
||||||
|
expect(result.conclusionConfidenceAfter).toBe("medium");
|
||||||
|
expect(result.confidenceCapReason).toBe(
|
||||||
|
"unresolved_direct_children_cap_conclusion",
|
||||||
|
);
|
||||||
expect(result.parentResolved).toBe(false);
|
expect(result.parentResolved).toBe(false);
|
||||||
expect(result.affectedAncestorIds).toContain(ids.parent);
|
expect(result.affectedAncestorIds).toContain(ids.parent);
|
||||||
expect(result.affectedAncestorIds).toContain(ids.ancestor);
|
expect(result.affectedAncestorIds).toContain(ids.ancestor);
|
||||||
@@ -276,7 +285,12 @@ describe("upward propagation", () => {
|
|||||||
);
|
);
|
||||||
expect(parentNode).toMatchObject({
|
expect(parentNode).toMatchObject({
|
||||||
status: "provisional",
|
status: "provisional",
|
||||||
confidence: "high",
|
confidence: "medium",
|
||||||
|
confidenceAssessment: {
|
||||||
|
evidenceConfidence: "high",
|
||||||
|
completenessStatus: "partial",
|
||||||
|
conclusionConfidence: "medium",
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
const ancestorNode = result.updatedSituationGraph.nodes.find(
|
const ancestorNode = result.updatedSituationGraph.nodes.find(
|
||||||
@@ -284,7 +298,12 @@ describe("upward propagation", () => {
|
|||||||
);
|
);
|
||||||
expect(ancestorNode).toMatchObject({
|
expect(ancestorNode).toMatchObject({
|
||||||
status: "provisional",
|
status: "provisional",
|
||||||
confidence: "high",
|
confidence: "low",
|
||||||
|
confidenceAssessment: {
|
||||||
|
evidenceConfidence: "low",
|
||||||
|
completenessStatus: "empty",
|
||||||
|
conclusionConfidence: "low",
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
const resolvedChild = result.updatedSituationGraph.nodes.find(
|
const resolvedChild = result.updatedSituationGraph.nodes.find(
|
||||||
@@ -387,6 +406,14 @@ describe("upward propagation", () => {
|
|||||||
expect(result.resolvedUnknownNodeIds).toContain(ids.parent);
|
expect(result.resolvedUnknownNodeIds).toContain(ids.parent);
|
||||||
expect(
|
expect(
|
||||||
result.updatedSituationGraph.nodes.find((node) => node.id === ids.parent),
|
result.updatedSituationGraph.nodes.find((node) => node.id === ids.parent),
|
||||||
).toMatchObject({ status: "resolved", confidence: "high" });
|
).toMatchObject({
|
||||||
|
status: "resolved",
|
||||||
|
confidence: "high",
|
||||||
|
confidenceAssessment: {
|
||||||
|
evidenceConfidence: "high",
|
||||||
|
completenessStatus: "complete",
|
||||||
|
conclusionConfidence: "high",
|
||||||
|
},
|
||||||
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user