feat: separate confidence from reasoning completeness

tests/graph/confidence-propagation.test.js

@@ -0,0 +1,179 @@
+import { describe, expect, it } from "vitest";
+import { applyValidatedProposal } from "@/lib/graph/apply-proposal.js";
+import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
+
+function makeFixture() {
+  const parent = makeNode({
+    id: "n-parent",
+    label: "Explanation for why revenue increased while cash fell",
+    description:
+      "Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
+    kind: "unknown",
+    status: "unknown",
+    confidence: "medium",
+  });
+  const children = [
+    makeNode({
+      id: "n-child-1",
+      label: "How the two observations were measured",
+      description: "Need evidence about the measure used for each observation.",
+      kind: "unknown",
+      status: "unknown",
+      confidence: "medium",
+      parentId: parent.id,
+    }),
+    makeNode({
+      id: "n-child-2",
+      label: "Whether the two observations reflect different timing",
+      description:
+        "Need to know whether the two observations reflect different timing.",
+      kind: "unknown",
+      status: "unknown",
+      confidence: "medium",
+      parentId: parent.id,
+    }),
+    makeNode({
+      id: "n-child-3",
+      label: "Possible change mainly affecting revenue",
+      description:
+        "Need to know whether a possible change mainly affected revenue.",
+      kind: "unknown",
+      status: "unknown",
+      confidence: "medium",
+      parentId: parent.id,
+    }),
+    makeNode({
+      id: "n-child-4",
+      label: "Possible one-off event during the period",
+      description:
+        "Need to know whether a possible one-off event happened during the period.",
+      kind: "unknown",
+      status: "unknown",
+      confidence: "medium",
+      parentId: parent.id,
+    }),
+  ];
+
+  return makeGraph({
+    centralStatement: "Revenue increased while cash fell.",
+    nodes: [parent, ...children],
+    edges: children.map((child, index) =>
+      makeEdge({
+        id: `e-${index + 1}`,
+        fromNodeId: child.id,
+        toNodeId: parent.id,
+        relationship: "depends_on",
+        description: `${child.label} feeds the parent.`,
+      }),
+    ),
+    activeUnknownNodeId: "n-child-1",
+    resolvedNodeIds: [],
+    currentSummary: "confidence propagation fixture",
+  });
+}
+
+function makeProposal({ resolvedIds, contradictedIds = [] }) {
+  return {
+    addedNodes: [
+      makeNode({
+        id: "n-anchor",
+        label: "Update anchor",
+        description:
+          "Anchor state introduced by the answer because the update must contain a meaningful change.",
+        kind: "state",
+        status: "known",
+        confidence: "low",
+      }),
+    ],
+    updatedNodes: [
+      ...resolvedIds.map((id) => ({
+        nodeId: id,
+        previousStatus: "unknown",
+        newStatus: "resolved",
+        previousValue: null,
+        newValue: `answer:${id}`,
+        reason: "resolved child",
+      })),
+      ...contradictedIds.map((id) => ({
+        nodeId: id,
+        previousStatus: "unknown",
+        newStatus: "contradicted",
+        previousValue: null,
+        newValue: `contradiction:${id}`,
+        reason: "contradictory child evidence",
+      })),
+    ],
+    addedEdges: [],
+    removedEdgeIds: [],
+    resolvedUnknownNodeIds: resolvedIds,
+    affectedNodeIds: [],
+    selectedQuestion: null,
+  };
+}
+
+describe("confidence propagation", () => {
+  it("one of four children resolved does not yield high conclusion confidence", () => {
+    const result = applyValidatedProposal({
+      situationGraph: makeFixture(),
+      proposal: makeProposal({ resolvedIds: ["n-child-1"] }),
+      previousQuestion:
+        "What evidence would clarify how the two observations were measured?",
+      answer: "Same accounting period and same management accounts.",
+    });
+
+    const parent = result.updatedSituationGraph.nodes.find(
+      (n) => n.id === "n-parent",
+    );
+    expect(parent.status).toBe("provisional");
+    expect(parent.confidence).toBe("medium");
+    expect(parent.confidenceAssessment).toEqual({
+      evidenceConfidence: "high",
+      completenessStatus: "partial",
+      conclusionConfidence: "medium",
+    });
+    expect(result.confidenceCapReason).toBe(
+      "unresolved_direct_children_cap_conclusion",
+    );
+  });
+
+  it("all children resolved with coherent evidence may yield high confidence", () => {
+    const result = applyValidatedProposal({
+      situationGraph: makeFixture(),
+      proposal: makeProposal({
+        resolvedIds: ["n-child-1", "n-child-2", "n-child-3", "n-child-4"],
+      }),
+      previousQuestion:
+        "What evidence would clarify how the two observations were measured?",
+      answer: "All direct child questions are answered.",
+    });
+
+    const parent = result.updatedSituationGraph.nodes.find(
+      (n) => n.id === "n-parent",
+    );
+    expect(parent.status).toBe("resolved");
+    expect(parent.confidenceAssessment).toEqual({
+      evidenceConfidence: "high",
+      completenessStatus: "complete",
+      conclusionConfidence: "high",
+    });
+  });
+
+  it("contradictory child evidence prevents high confidence", () => {
+    const result = applyValidatedProposal({
+      situationGraph: makeFixture(),
+      proposal: makeProposal({
+        resolvedIds: ["n-child-1"],
+        contradictedIds: ["n-child-2"],
+      }),
+      previousQuestion:
+        "What evidence would clarify how the two observations were measured?",
+      answer: "One child resolved, another contradicted.",
+    });
+
+    const parent = result.updatedSituationGraph.nodes.find(
+      (n) => n.id === "n-parent",
+    );
+    expect(parent.confidenceAssessment.conclusionConfidence).toBe("low");
+    expect(result.confidenceCapReason).toBe("contradictory_direct_children");
+  });
+});

tests/graph/upward-propagation.test.js

@@ -255,7 +255,16 @@ describe("upward propagation", () => {
       expect(result.parentStatusBefore).toBe("unknown");
       expect(result.parentStatusAfter).toBe("provisional");
       expect(result.parentConfidenceBefore).toBe("medium");
-      expect(result.parentConfidenceAfter).toBe("high");
+      expect(result.parentConfidenceAfter).toBe("medium");
+      expect(result.evidenceConfidenceBefore).toBe("medium");
+      expect(result.evidenceConfidenceAfter).toBe("high");
+      expect(result.completenessBefore).toBe("empty");
+      expect(result.completenessAfter).toBe("partial");
+      expect(result.conclusionConfidenceBefore).toBe("low");
+      expect(result.conclusionConfidenceAfter).toBe("medium");
+      expect(result.confidenceCapReason).toBe(
+        "unresolved_direct_children_cap_conclusion",
+      );
       expect(result.parentResolved).toBe(false);
       expect(result.affectedAncestorIds).toContain(ids.parent);
       expect(result.affectedAncestorIds).toContain(ids.ancestor);
@@ -276,7 +285,12 @@ describe("upward propagation", () => {
       );
       expect(parentNode).toMatchObject({
         status: "provisional",
-        confidence: "high",
+        confidence: "medium",
+        confidenceAssessment: {
+          evidenceConfidence: "high",
+          completenessStatus: "partial",
+          conclusionConfidence: "medium",
+        },
       });
 
       const ancestorNode = result.updatedSituationGraph.nodes.find(
@@ -284,7 +298,12 @@ describe("upward propagation", () => {
       );
       expect(ancestorNode).toMatchObject({
         status: "provisional",
-        confidence: "high",
+        confidence: "low",
+        confidenceAssessment: {
+          evidenceConfidence: "low",
+          completenessStatus: "empty",
+          conclusionConfidence: "low",
+        },
       });
 
       const resolvedChild = result.updatedSituationGraph.nodes.find(
@@ -387,6 +406,14 @@ describe("upward propagation", () => {
     expect(result.resolvedUnknownNodeIds).toContain(ids.parent);
     expect(
       result.updatedSituationGraph.nodes.find((node) => node.id === ids.parent),
-    ).toMatchObject({ status: "resolved", confidence: "high" });
+    ).toMatchObject({
+      status: "resolved",
+      confidence: "high",
+      confidenceAssessment: {
+        evidenceConfidence: "high",
+        completenessStatus: "complete",
+        conclusionConfidence: "high",
+      },
+    });
   });
 });