From b1c633ba5ce7bee9960431ef3d42029941f2c0aa Mon Sep 17 00:00:00 2001 From: robbond Date: Sun, 2 Aug 2026 19:03:07 +0100 Subject: [PATCH] feat: back next questions with explicit graph unknowns --- docs/v0.6-comparability-experiment.md | 4 + lib/graph/apply-proposal.js | 184 +++++++++++++++++++++++--- lib/graph/orchestrator.js | 17 +++ lib/graph/question-formulator.js | 14 +- tests/graph/apply-proposal.test.js | 61 +++++++++ tests/graph/orchestrator.test.js | 6 + tests/ui/scenario-form.test.jsx | 64 ++++++--- 7 files changed, 310 insertions(+), 40 deletions(-) diff --git a/docs/v0.6-comparability-experiment.md b/docs/v0.6-comparability-experiment.md index bc3da2d..b19507f 100644 --- a/docs/v0.6-comparability-experiment.md +++ b/docs/v0.6-comparability-experiment.md @@ -42,3 +42,7 @@ The repeated pattern appeared in four scenarios, so a small pre-contradiction co A comparison question is useful only if its answer advances the reasoning stage rather than merely adding more text. In the revenue-versus-cash scenario, the first question now confirms whether the figures are comparable, and the answer resolves that existing uncertainty instead of creating a parallel note. After that update, the engine progresses from comparability assessment to cautious relationship assessment and can select one broad non-expert follow-up question. + +Every justified next question should correspond to an explicit unresolved graph node. + +The earlier fallback-only path has now been removed from the normal successful progression. After comparability is resolved and a further investigation question is justified, the engine creates or reuses an explicit unresolved reasoning unknown and lets deterministic selection and question formulation proceed through the standard graph pipeline. A fallback is now only acceptable as an explicit failure case, not as the normal source of the next question. diff --git a/lib/graph/apply-proposal.js b/lib/graph/apply-proposal.js index 5529d97..4d670e6 100644 --- a/lib/graph/apply-proposal.js +++ b/lib/graph/apply-proposal.js @@ -1,11 +1,15 @@ import { describeGraph } from "./builder.js"; import { buildReasoningState, + classifyObservationRelationship, COMPARABILITY_REASONING_NODE_ID, formulateQuestion, - formulateTieResolutionQuestion, } from "./question-formulator.js"; -import { graphUpdateSchema, situationGraphSchema } from "./schema.js"; +import { + graphUpdateSchema, + makeNodeId, + situationGraphSchema, +} from "./schema.js"; import { applyGraphUpdate, detectDuplicateNodeIds, @@ -428,6 +432,124 @@ function buildChangesApplied(proposal, affectedNodeIds) { }; } +function buildEmergentReasoningUnknownLabel(graph) { + const central = String(graph?.centralStatement || "these observations") + .trim() + .replace(/[.?!:;]+$/g, ""); + return `Explanation for why ${central}`; +} + +function findEquivalentEmergentUnknown(graph, label, description) { + const targetId = makeNodeId(label); + const targetTexts = [normaliseText(label), normaliseText(description)].filter( + Boolean, + ); + + return (graph.nodes || []).find((node) => { + if ( + node.kind !== "unknown" || + (graph.resolvedNodeIds || []).includes(node.id) + ) { + return false; + } + + if (node.id === targetId) { + return true; + } + + const nodeTexts = [ + normaliseText(node.label), + normaliseText(node.description), + ].filter(Boolean); + + return targetTexts.some((text) => nodeTexts.includes(text)); + }); +} + +function buildEmergentReasoningUnknown(graph, relationshipAssessment) { + if (!relationshipAssessment?.relationshipAssessed) { + return null; + } + + if (!relationshipAssessment.questionRequired) { + return null; + } + + if ( + ![ + "potentially_related", + "insufficient_information", + "contradictory", + ].includes(relationshipAssessment.relationshipStatus) + ) { + return null; + } + + const label = buildEmergentReasoningUnknownLabel(graph); + const description = + "Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship."; + const existingNode = findEquivalentEmergentUnknown(graph, label, description); + if (existingNode) { + return { + created: false, + node: existingNode, + edges: [], + reason: + "Reused an existing unresolved reasoning unknown for the next investigation stage.", + }; + } + + const observationNodes = (graph.nodes || []).filter( + (node) => node.kind === "observation" && node.status === "supported", + ); + const relationshipNode = (graph.nodes || []).find( + (node) => node.kind === "relationship" && node.status === "supported", + ); + const nodeId = makeNodeId(label); + const relatedNodeIds = relationshipNode + ? [relationshipNode.id] + : observationNodes.slice(0, 2).map((node) => node.id); + + if (relatedNodeIds.length === 0) { + return null; + } + + const node = { + id: nodeId, + label, + description, + kind: "unknown", + status: "unknown", + confidence: "medium", + value: null, + unit: null, + evidenceIds: [], + dependsOn: relatedNodeIds, + affects: [], + parentId: relationshipNode?.id ?? null, + childIds: [], + }; + + const edges = relatedNodeIds.map((relatedNodeId) => ({ + id: `e-${relatedNodeId.slice(0, 6)}-${nodeId.slice(0, 6)}`, + fromNodeId: relatedNodeId, + toNodeId: nodeId, + relationship: + relationshipNode?.id === relatedNodeId ? "depends_on" : "other", + confidence: "medium", + description: + "This unresolved explanation arises from the now-assessed relationship between the observations.", + })); + + return { + created: true, + node, + edges, + reason: + "Created a new unresolved reasoning unknown so the next justified question is backed by the graph.", + }; +} + function isComparabilityQuestion(question) { const text = String(question || "").toLowerCase(); return ( @@ -643,6 +765,36 @@ export function applyValidatedProposal({ resolvedUnknownNodeIds: validatedProposal.resolvedUnknownNodeIds, }); + const provisionalApplied = applyGraphUpdate(graphSnapshot, proposalSnapshot); + if (!provisionalApplied.success) { + return { + success: false, + stage: "application", + errors: provisionalApplied.errors, + }; + } + const provisionalGraph = { + ...graphSnapshot, + nodes: provisionalApplied.nodes, + edges: provisionalApplied.edges, + resolvedNodeIds: provisionalApplied.resolvedNodeIds, + }; + provisionalGraph.reasoningState = buildReasoningState( + provisionalGraph, + reasoningResolution.reasoningStateOverride, + ); + const relationshipAssessment = + classifyObservationRelationship(provisionalGraph); + const emergentReasoningUnknown = buildEmergentReasoningUnknown( + provisionalGraph, + relationshipAssessment, + ); + + if (emergentReasoningUnknown?.created) { + proposalSnapshot.addedNodes.push(emergentReasoningUnknown.node); + proposalSnapshot.addedEdges.push(...emergentReasoningUnknown.edges); + } + const applied = applyGraphUpdate(graphSnapshot, proposalSnapshot); if (!applied.success) { return { @@ -738,7 +890,8 @@ export function applyValidatedProposal({ ? { nodeId: null, tiedCandidateIds: deterministicSelection.tiedCandidateIds, - ...formulateTieResolutionQuestion({ graph: updatedSituationGraph }), + question: null, + reason: deterministicSelection.reason, } : deterministicSelection?.status === "selected" ? { @@ -749,21 +902,7 @@ export function applyValidatedProposal({ strategy: formulatedQuestion?.strategy, investigationStrategy: formulatedQuestion?.investigationStrategy, } - : (() => { - const relationshipFallback = formulateTieResolutionQuestion({ - graph: updatedSituationGraph, - }); - return relationshipFallback?.question - ? { - nodeId: null, - question: relationshipFallback.question, - reason: relationshipFallback.reason, - strategy: relationshipFallback.strategy, - investigationStrategy: - relationshipFallback.investigationStrategy, - } - : null; - })(); + : null; const resultGraphValidation = situationGraphSchema.safeParse( updatedSituationGraph, @@ -809,14 +948,17 @@ export function applyValidatedProposal({ return { success: true, updatedSituationGraph, - graphUpdate: validatedProposal, + graphUpdate: proposalSnapshot, affectedNodeIds, - resolvedUnknownNodeIds: validatedProposal.resolvedUnknownNodeIds, + resolvedUnknownNodeIds: proposalSnapshot.resolvedUnknownNodeIds, resolvedReasoningNodeIds: reasoningResolution.resolvedReasoningNodeIds, + emergentReasoningNodeCreated: Boolean(emergentReasoningUnknown?.created), + emergentReasoningNodeId: emergentReasoningUnknown?.node?.id ?? null, + emergentReasoningNodeReason: emergentReasoningUnknown?.reason ?? null, previousActiveUnknownNodeId, newActiveUnknownNodeId, selectedQuestion: finalSelectedQuestion, - changesApplied: buildChangesApplied(validatedProposal, affectedNodeIds), + changesApplied: buildChangesApplied(proposalSnapshot, affectedNodeIds), graphReferenceValidation: resultReferenceValidation, previousReasoningState: reasoningResolution.previousReasoningState, reasoningState: nextReasoningState, diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index d2f4ccf..ce6f16d 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -89,6 +89,9 @@ function buildUpdateDiagnostics({ previousReasoningState, reasoningState, resolvedReasoningNodeIds, + emergentReasoningNodeCreated, + emergentReasoningNodeId, + emergentReasoningNodeReason, }) { return { promptVersion: promptVersion ?? "v0.4", @@ -114,6 +117,9 @@ function buildUpdateDiagnostics({ reasoningStagesBefore: previousReasoningState?.reasoningStages ?? [], reasoningStagesAfter: reasoningState?.reasoningStages ?? [], resolvedReasoningNodeIds: resolvedReasoningNodeIds ?? [], + emergentReasoningNodeCreated: emergentReasoningNodeCreated ?? false, + emergentReasoningNodeId: emergentReasoningNodeId ?? null, + emergentReasoningNodeReason: emergentReasoningNodeReason ?? null, unknownSelectionExplanation: unknownSelectionExplanation ?? null, }; } @@ -363,6 +369,9 @@ async function updateCaseWithDependencies(body, dependencies = {}) { previousReasoningState: buildReasoningState(situationGraph), reasoningState: buildReasoningState(situationGraph), resolvedReasoningNodeIds: [], + emergentReasoningNodeCreated: false, + emergentReasoningNodeId: null, + emergentReasoningNodeReason: null, unknownSelectionExplanation: explainUnknownSelection( situationGraph, situationGraph.resolvedNodeIds || [], @@ -400,6 +409,11 @@ async function updateCaseWithDependencies(body, dependencies = {}) { previousReasoningState: applicationResult.previousReasoningState, reasoningState: applicationResult.reasoningState, resolvedReasoningNodeIds: applicationResult.resolvedReasoningNodeIds, + emergentReasoningNodeCreated: + applicationResult.emergentReasoningNodeCreated, + emergentReasoningNodeId: applicationResult.emergentReasoningNodeId, + emergentReasoningNodeReason: + applicationResult.emergentReasoningNodeReason, unknownSelectionExplanation: buildUnknownSelectionDiagnostics( applicationResult.updatedSituationGraph, applicationResult.updatedSituationGraph.resolvedNodeIds || [], @@ -424,6 +438,9 @@ async function updateCaseWithDependencies(body, dependencies = {}) { previousReasoningState: buildReasoningState(situationGraph), reasoningState: buildReasoningState(situationGraph), resolvedReasoningNodeIds: [], + emergentReasoningNodeCreated: false, + emergentReasoningNodeId: null, + emergentReasoningNodeReason: null, unknownSelectionExplanation: buildUnknownSelectionDiagnostics( situationGraph, situationGraph.resolvedNodeIds || [], diff --git a/lib/graph/question-formulator.js b/lib/graph/question-formulator.js index 7497430..dcef219 100644 --- a/lib/graph/question-formulator.js +++ b/lib/graph/question-formulator.js @@ -498,6 +498,16 @@ function buildBroadInvestigationQuestion(graph) { return `What changed during that period that could help explain why ${central}?`; } +function isRelationshipExplanationUnknown(node, graph) { + const text = normaliseText(`${node?.label || ""} ${node?.description || ""}`); + return ( + collectObservationNodes(graph).length >= 2 && + /\b(explain|explanation|divergence|moved differently|difference between|change or event|what changed|why the observations)/.test( + text, + ) + ); +} + export function formulateTieResolutionQuestion({ graph }) { const comparability = assessComparability(graph); if (comparability.comparabilityStatus === "uncertain") { @@ -909,7 +919,9 @@ export function formulateQuestion({ node, graph, context = {} }) { let question = investigationStrategy ? buildQuestionFromStrategy(investigationStrategy) - : buildNeutralClarificationQuestion(extractMeaning(node)); + : isRelationshipExplanationUnknown(node, graph) + ? buildBroadInvestigationQuestion(graph) + : buildNeutralClarificationQuestion(extractMeaning(node)); question = sanitizeQuestionText(question); diff --git a/tests/graph/apply-proposal.test.js b/tests/graph/apply-proposal.test.js index c97af00..42804b9 100644 --- a/tests/graph/apply-proposal.test.js +++ b/tests/graph/apply-proposal.test.js @@ -1133,6 +1133,9 @@ describe("applyValidatedProposal", () => { expect(result.resolvedReasoningNodeIds).toEqual([ "reasoning:comparability", ]); + expect(result.emergentReasoningNodeCreated).toBe(true); + expect(result.emergentReasoningNodeId).toBeTruthy(); + expect(result.emergentReasoningNodeReason).toContain("backed by the graph"); expect(result.previousReasoningState.comparabilityStatus).toBe("uncertain"); expect(result.reasoningState).toMatchObject({ comparabilityStatus: "confirmed", @@ -1142,6 +1145,7 @@ describe("applyValidatedProposal", () => { expect(result.reasoningState.comparabilityEvidence).toEqual([ comparabilityUnknownId, ]); + expect(result.selectedQuestion?.nodeId).toBe(result.newActiveUnknownNodeId); expect(result.selectedQuestion?.question).toMatch( /^What changed during that period that could help explain why /, ); @@ -1163,6 +1167,27 @@ describe("applyValidatedProposal", () => { "The observations concern connected business signals but do not establish a direct contradiction or cause.", }, ]); + const emergentNode = result.updatedSituationGraph.nodes.find( + (node) => node.id === result.emergentReasoningNodeId, + ); + expect(emergentNode).toMatchObject({ + kind: "unknown", + status: "unknown", + confidence: "medium", + }); + expect(emergentNode.description.toLowerCase()).toContain("because"); + expect( + result.updatedSituationGraph.edges.filter( + (edge) => edge.toNodeId === result.emergentReasoningNodeId, + ), + ).not.toEqual([]); + expect( + result.updatedSituationGraph.edges.some( + (edge) => + edge.toNodeId === result.emergentReasoningNodeId && + edge.relationship === "causes", + ), + ).toBe(false); expect( JSON.stringify( result.updatedSituationGraph.nodes.find( @@ -1171,4 +1196,40 @@ describe("applyValidatedProposal", () => { ), ).toBe(originalUnrelatedNode); }); + + it("reuses an equivalent existing unresolved reasoning unknown instead of creating a duplicate", () => { + const { graph, proposal } = makeComparabilityUpdateFixture(); + graph.nodes.push( + makeNode({ + id: "n-existing-explanation", + label: + "Explanation for why Revenue increased by 18%, but cash in the bank fell over the same period", + description: + "Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.", + kind: "unknown", + status: "unknown", + confidence: "medium", + }), + ); + + const result = applyValidatedProposal({ + situationGraph: graph, + proposal, + previousQuestion: + "Were these figures measured on the same basis and at the same scale?", + answer: + "Yes. Both figures cover the same accounting period and are taken from the same management accounts.", + }); + + expect(result.success).toBe(true); + expect(result.emergentReasoningNodeCreated).toBe(false); + expect(result.emergentReasoningNodeId).toBe("n-existing-explanation"); + expect(result.newActiveUnknownNodeId).toBe("n-existing-explanation"); + expect(result.selectedQuestion?.nodeId).toBe("n-existing-explanation"); + expect( + result.updatedSituationGraph.nodes.filter( + (node) => node.label === graph.nodes.at(-1).label, + ), + ).toHaveLength(1); + }); }); diff --git a/tests/graph/orchestrator.test.js b/tests/graph/orchestrator.test.js index 9e6d161..50b916d 100644 --- a/tests/graph/orchestrator.test.js +++ b/tests/graph/orchestrator.test.js @@ -1046,7 +1046,12 @@ describe("lib/graph/orchestrator startCase", () => { relationshipStatus: "potentially_related", relationshipAssessed: true, resolvedReasoningNodeIds: ["reasoning:comparability"], + emergentReasoningNodeCreated: true, }); + expect(result.diagnostics.emergentReasoningNodeId).toBeTruthy(); + expect(result.diagnostics.emergentReasoningNodeReason).toContain( + "backed by the graph", + ); expect(result.diagnostics.reasoningStagesBefore).toEqual([ { stage: "comparability", @@ -1074,6 +1079,7 @@ describe("lib/graph/orchestrator startCase", () => { "The observations concern connected business signals but do not establish a direct contradiction or cause.", }, ]); + expect(result.selectedQuestion?.nodeId).toBe(result.newActiveUnknownNodeId); expect(result.selectedQuestion?.question).toMatch( /^What changed during that period that could help explain why /, ); diff --git a/tests/ui/scenario-form.test.jsx b/tests/ui/scenario-form.test.jsx index 04e0c05..a1955a7 100644 --- a/tests/ui/scenario-form.test.jsx +++ b/tests/ui/scenario-form.test.jsx @@ -115,32 +115,46 @@ function makeUpdateSuccess(overrides = {}) { }, { id: "n-next-unknown", - label: "Commercial value definition", - description: "Need a definition because the decision depends on it.", + label: + "Explanation for why revenue increased by 18%, but cash in the bank fell over the same period", + description: + "Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.", kind: "unknown", status: "unknown", - confidence: "high", + confidence: "medium", value: null, unit: null, }, ], - edges: [], + edges: [ + { + id: "e-rel-next", + fromNodeId: "n-conclusion", + toNodeId: "n-next-unknown", + relationship: "depends_on", + confidence: "medium", + description: + "This unresolved explanation arises from the now-assessed relationship between the observations.", + }, + ], }, proposal: { addedNodes: [ { id: "n-next-unknown", - label: "Commercial value definition", - description: "Need a definition because the decision depends on it.", + label: + "Explanation for why revenue increased by 18%, but cash in the bank fell over the same period", + description: + "Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.", kind: "unknown", status: "unknown", - confidence: "high", + confidence: "medium", value: null, unit: null, evidenceIds: [], - dependsOn: [], + dependsOn: ["n-conclusion"], affects: [], - parentId: null, + parentId: "n-conclusion", childIds: [], }, ], @@ -153,19 +167,27 @@ function makeUpdateSuccess(overrides = {}) { affectedNodeIds: ["n-conclusion"], selectedQuestion: { nodeId: "n-next-unknown", - question: "How should commercial value be defined for this decision?", - reason: "A narrower consequential uncertainty remains.", + question: + "What changed during that period that could help explain why revenue increased by 18%, but cash in the bank fell over the same period?", + reason: + "Formulated as a neutral clarification question because no narrower investigation strategy clearly applied.", }, }, selectedQuestion: { nodeId: "n-next-unknown", - question: "How should commercial value be defined for this decision?", - reason: "A narrower consequential uncertainty remains.", + question: + "What changed during that period that could help explain why revenue increased by 18%, but cash in the bank fell over the same period?", + reason: + "Formulated as a neutral clarification question because no narrower investigation strategy clearly applied.", }, affectedNodeIds: ["n-conclusion"], resolvedUnknownNodeIds: ["n-unknown"], previousActiveUnknownNodeId: "n-unknown", newActiveUnknownNodeId: "n-next-unknown", + emergentReasoningNodeCreated: true, + emergentReasoningNodeId: "n-next-unknown", + emergentReasoningNodeReason: + "Created a new unresolved reasoning unknown so the next justified question is backed by the graph.", previousReasoningState: { comparabilityStatus: "uncertain", reasoningStages: [ @@ -404,7 +426,9 @@ describe("graph-backed UI rendering", () => { ); expect(html).toContain("Newly surfaced unknowns"); - expect(html).toContain("Commercial value definition"); + expect(html).toContain( + "Explanation for why revenue increased by 18%, but cash in the bank fell over the same period", + ); }); it("affected nodes render", () => { @@ -432,7 +456,7 @@ describe("graph-backed UI rendering", () => { ); expect(html).toContain( - "How should commercial value be defined for this decision?", + "What changed during that period that could help explain why revenue increased by 18%, but cash in the bank fell over the same period?", ); }); @@ -469,7 +493,9 @@ describe("graph-backed UI rendering", () => { expect(html).toContain("Previous active unknown"); expect(html).toContain("Complaint rate denominator"); expect(html).toContain("New active unknown"); - expect(html).toContain("Commercial value definition"); + expect(html).toContain( + "Explanation for why revenue increased by 18%, but cash in the bank fell over the same period", + ); }); it("successful update renders prior and new state together", () => { @@ -488,7 +514,7 @@ describe("graph-backed UI rendering", () => { expect(html).toContain("New active unknown"); expect(html).toContain("Next question"); expect(html).toContain( - "How should commercial value be defined for this decision?", + "What changed during that period that could help explain why revenue increased by 18%, but cash in the bank fell over the same period?", ); }); @@ -543,7 +569,9 @@ describe("graph-backed UI rendering", () => { />, ); - expect(html).toContain("How should commercial value be defined for this decision?"); + expect(html).toContain( + "What changed during that period that could help explain why revenue increased by 18%, but cash in the bank fell over the same period?", + ); }); it("raw ids remain only in collapsed proposal details", () => {