From 5ef9710293e5e6281e4197b06edd23cd7a787325 Mon Sep 17 00:00:00 2001 From: robbond Date: Sun, 2 Aug 2026 15:10:49 +0100 Subject: [PATCH 01/17] Implemented Investigation Strategy --- components/diagnostics-view.jsx | 8 + lib/graph/apply-proposal.js | 1 + lib/graph/orchestrator.js | 8 + lib/graph/question-formulator.js | 239 +++++++++++------- .../question-priority-generalisation.js | 22 +- tests/graph/question-formulator.test.js | 126 +++++++-- .../question-priority-generalisation.test.js | 10 +- 7 files changed, 293 insertions(+), 121 deletions(-) diff --git a/components/diagnostics-view.jsx b/components/diagnostics-view.jsx index 8049c26..352795c 100644 --- a/components/diagnostics-view.jsx +++ b/components/diagnostics-view.jsx @@ -80,6 +80,14 @@ export default function DiagnosticsView({ result }) { ? `${validationIcons.valid} valid` : `${validationIcons.invalid} invalid`, }, + { + label: "Investigation strategy", + value: + diagnostics.investigationStrategy?.key || + diagnostics.investigationStrategy || + result.selectedQuestion?.strategy || + "?", + }, ]; const errors = [ diff --git a/lib/graph/apply-proposal.js b/lib/graph/apply-proposal.js index da727f8..ff82ce3 100644 --- a/lib/graph/apply-proposal.js +++ b/lib/graph/apply-proposal.js @@ -656,6 +656,7 @@ export function applyValidatedProposal({ situationGraph, proposal }) { formulatedQuestion?.question || deterministicSelection.question, reason: formulatedQuestion?.reason || deterministicSelection.reason, strategy: formulatedQuestion?.strategy, + investigationStrategy: formulatedQuestion?.investigationStrategy, } : null; diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index fc97ac8..cbbc894 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -53,6 +53,7 @@ function buildUpdateDiagnostics({ normalisationsApplied, graph, graphReferenceValidation, + selectedQuestion, }) { return { promptVersion: promptVersion ?? "v0.4", @@ -66,6 +67,10 @@ function buildUpdateDiagnostics({ errors: [], }, normalisationsApplied: normalisationsApplied ?? [], + investigationStrategy: + selectedQuestion?.investigationStrategy ?? + selectedQuestion?.strategy ?? + null, }; } @@ -284,6 +289,7 @@ async function updateCaseWithDependencies(body, dependencies = {}) { normalisationsApplied: parsedProposal.normalisationsApplied, graph: situationGraph, graphReferenceValidation: graphReferenceValidation, + selectedQuestion: null, }), }, statusCode: @@ -313,6 +319,7 @@ async function updateCaseWithDependencies(body, dependencies = {}) { normalisationsApplied: parsedProposal.normalisationsApplied, graph: applicationResult.updatedSituationGraph, graphReferenceValidation: applicationResult.graphReferenceValidation, + selectedQuestion: applicationResult.selectedQuestion, }), }; } @@ -328,6 +335,7 @@ async function updateCaseWithDependencies(body, dependencies = {}) { normalisationsApplied: parsedProposal.normalisationsApplied, graph: situationGraph, graphReferenceValidation, + selectedQuestion: null, }), }; } diff --git a/lib/graph/question-formulator.js b/lib/graph/question-formulator.js index da164ae..60670b5 100644 --- a/lib/graph/question-formulator.js +++ b/lib/graph/question-formulator.js @@ -139,7 +139,41 @@ function toGerundPhrase(phrase) { return [gerund, ...rest].join(" "); } -function detectStrategy({ node, graph, relatedNodes, combinedText, meaning }) { +function buildInvestigationStrategy({ + key, + reason, + node, + graph, + relatedNodes, + meaning, + actionPhrase, +}) { + return { + key, + reason, + nodeId: node?.id ?? null, + nodeLabel: node?.label ?? null, + meaning, + actionPhrase, + relatedNodeIds: relatedNodes.map((relatedNode) => relatedNode.id), + centralStatement: graph?.centralStatement ?? null, + }; +} + +export function selectInvestigationStrategy({ node, graph, context = {} }) { + const relatedNodes = collectRelatedNodes(node, graph); + const meaning = extractMeaning(node); + const combinedText = [ + node?.label, + node?.description, + ...relatedNodes.map((relatedNode) => relatedNode.label), + ...relatedNodes.map((relatedNode) => relatedNode.description), + graph?.centralStatement, + ...(context.resolvedValues || []), + ] + .filter(Boolean) + .join(" "); + const text = normaliseText(combinedText); const nodeText = normaliseText( `${node?.label || ""} ${node?.description || ""}`, @@ -172,21 +206,22 @@ function detectStrategy({ node, graph, relatedNodes, combinedText, meaning }) { nodeText, ); - if (/\b(customer|user|buyer|stakeholder|recipient|audience)\b/.test(text)) { - return { strategy: "actor/customer", meaning, actionPhrase }; - } - const hasBaselineLanguage = /\b(before|previous|baseline|prior|comparable state)\b/.test(text); const hasPrimaryBaselineLanguage = /\b(before|previous|baseline|prior|comparable state)\b/.test(nodeText); if (hasBaselineLanguage && hasPrimaryBaselineLanguage) { - return { strategy: "baseline", meaning, actionPhrase }; - } - - if (/\b(when|timing|timeline|duration|sequence|milestone)\b/.test(text)) { - return { strategy: "transition/timing", meaning, actionPhrase }; + return buildInvestigationStrategy({ + key: "baseline_reconstruction", + reason: + "Selected because the unknown explicitly references a missing previous or baseline state.", + node, + graph, + relatedNodes, + meaning, + actionPhrase, + }); } const hasDefinitionLanguage = @@ -206,84 +241,116 @@ function detectStrategy({ node, graph, relatedNodes, combinedText, meaning }) { /\b(metric|measure|measurable|roi|revenue projection|benchmark)\b/.test( text, ); + const hasEvidenceLanguage = + /\b(evidence|proof|validate|validation|signal|demand)\b/.test(text) || + node?.kind === "reported_claim" || + node?.kind === "conclusion" || + /\b(claim|assertion|true|false)\b/.test(text); + const hasContradictionLanguage = + /\b(contradiction|contradict|conflict|inconsistent|inconsistency|disagree|mismatch)\b/.test( + `${text} ${relatedText}`, + ) || + relatedNodes.some( + (relatedNode) => + relatedNode.status === "contradicted" || + relatedNode.kind === "conclusion", + ); - if (hasDecisionValueLanguage && hasMeasurementLanguage) { - return { strategy: "measurement", meaning, actionPhrase }; - } - - if (hasPrimaryDefinitionLanguage) { - return { strategy: "definition", meaning, actionPhrase }; + if (hasPrimaryDefinitionLanguage || hasDefinitionLanguage) { + return buildInvestigationStrategy({ + key: "definition", + reason: + "Selected because the unknown is primarily about clarifying what a term means in this case.", + node, + graph, + relatedNodes, + meaning, + actionPhrase, + }); } if (hasDecisionValueLanguage || hasCriteriaLanguage) { - return { strategy: "decision criterion", meaning, actionPhrase }; + return buildInvestigationStrategy({ + key: "decision_threshold", + reason: + "Selected because the unknown determines the threshold for making or justifying a decision.", + node, + graph, + relatedNodes, + meaning, + actionPhrase, + }); } - if (hasConstraintLanguage && hasPrimaryConstraintLanguage) { - return { strategy: "constraint", meaning, actionPhrase }; + if (hasPrimaryBaselineLanguage || hasBaselineLanguage) { + return buildInvestigationStrategy({ + key: "baseline_reconstruction", + reason: + "Selected because reconstructing the prior state is the most direct way to resolve the unknown.", + node, + graph, + relatedNodes, + meaning, + actionPhrase, + }); } - if (hasDefinitionLanguage) { - return { strategy: "definition", meaning, actionPhrase }; + if (hasContradictionLanguage) { + return buildInvestigationStrategy({ + key: "contradiction_resolution", + reason: + "Selected because the graph context indicates conflicting claims or inconsistent states that must be reconciled.", + node, + graph, + relatedNodes, + meaning, + actionPhrase, + }); } - if (hasBaselineLanguage) { - return { strategy: "baseline", meaning, actionPhrase }; + if (hasEvidenceLanguage || hasMeasurementLanguage || hasConstraintLanguage) { + return buildInvestigationStrategy({ + key: "evidence_gathering", + reason: + hasConstraintLanguage && hasPrimaryConstraintLanguage + ? "Selected because evidence about the practical limiting factor is needed before the unknown can be resolved." + : "Selected because resolving the unknown requires evidence, signals, or measurable confirmation.", + node, + graph, + relatedNodes, + meaning, + actionPhrase, + }); } - if (hasConstraintLanguage) { - return { strategy: "constraint", meaning, actionPhrase }; - } - - if (/\b(evidence|proof|validate|validation|signal|demand)\b/.test(text)) { - return { strategy: "evidence", meaning, actionPhrase }; - } - - if (hasMeasurementLanguage) { - return { strategy: "measurement", meaning, actionPhrase }; - } - - if ( - /\b(objective|goal|outcome|problem|job to be done|benefit)\b/.test(text) - ) { - return { strategy: "objective", meaning, actionPhrase }; - } - - if ( - node?.kind === "reported_claim" || - node?.kind === "conclusion" || - /\b(claim|assertion|true|false)\b/.test(text) - ) { - return { strategy: "evidence", meaning, actionPhrase }; - } - - return { strategy: "generic clarification", meaning, actionPhrase }; + return buildInvestigationStrategy({ + key: "definition", + reason: + "Selected as the deterministic fallback because clarifying the exact meaning of the unknown is the narrowest first step.", + node, + graph, + relatedNodes, + meaning, + actionPhrase, + }); } -function buildQuestion({ strategy, meaning, actionPhrase }) { - switch (strategy) { - case "decision criterion": - return actionPhrase - ? `What outcome would demonstrate enough value to justify ${toGerundPhrase(actionPhrase)}?` +function buildQuestionFromStrategy(strategy) { + switch (strategy.key) { + case "decision_threshold": + return strategy.actionPhrase + ? `What outcome would demonstrate enough value to justify ${toGerundPhrase(strategy.actionPhrase)}?` : "What outcome would be sufficient to justify this decision?"; case "definition": - return `What does ${meaning} mean in this situation?`; - case "evidence": - return `What evidence would show whether ${meaning} is true?`; - case "baseline": - return `What was the comparable state before ${meaning}?`; - case "actor/customer": - return "Who experiences the problem or receives the value in this situation?"; - case "objective": - return "What outcome is this decision or effort meant to achieve?"; - case "constraint": - return "What constraint most limits the available options in this situation?"; - case "measurement": - return `What measure would determine whether ${meaning} is sufficient?`; - case "transition/timing": - return `When does ${meaning} become relevant in the decision or change?`; + return `What does ${strategy.meaning} mean in this situation?`; + case "evidence_gathering": + return `What evidence would show whether ${strategy.meaning} is true?`; + case "baseline_reconstruction": + return `What was the comparable state before ${strategy.meaning}?`; + case "contradiction_resolution": + return `What fact would resolve the contradiction about ${strategy.meaning}?`; default: - return `What specific fact would resolve whether ${meaning} is true?`; + return `What specific fact would resolve whether ${strategy.meaning} is true?`; } } @@ -332,36 +399,22 @@ function validateFormulatedQuestion(question, meaning) { } export function formulateQuestion({ node, graph, context = {} }) { - const relatedNodes = collectRelatedNodes(node, graph); - const meaning = extractMeaning(node); - const combinedText = [ - node?.label, - node?.description, - ...relatedNodes.map((relatedNode) => relatedNode.label), - ...relatedNodes.map((relatedNode) => relatedNode.description), - graph?.centralStatement, - ...(context.resolvedValues || []), - ] - .filter(Boolean) - .join(" "); - - const detected = detectStrategy({ + const investigationStrategy = selectInvestigationStrategy({ node, graph, - relatedNodes, - combinedText, - meaning, + context, }); - let question = buildQuestion(detected); + let question = buildQuestionFromStrategy(investigationStrategy); - if (!validateFormulatedQuestion(question, meaning)) { - question = `What evidence would resolve whether ${meaning} is true?`; + if (!validateFormulatedQuestion(question, investigationStrategy.meaning)) { + question = `What evidence would resolve whether ${investigationStrategy.meaning} is true?`; } return { question, - reason: `Formulated from graph context using the ${detected.strategy} strategy.`, - strategy: detected.strategy, + reason: `Formulated from graph context using the ${investigationStrategy.key} investigation strategy.`, + strategy: investigationStrategy.key, + investigationStrategy, }; } diff --git a/tests/fixtures/question-priority-generalisation.js b/tests/fixtures/question-priority-generalisation.js index d3cb817..09834e2 100644 --- a/tests/fixtures/question-priority-generalisation.js +++ b/tests/fixtures/question-priority-generalisation.js @@ -67,7 +67,11 @@ export const questionPriorityGeneralisationFixtures = [ "hire-bottleneck", ], prohibitedFirstTopics: ["salary", "job advert", "programming language"], - acceptableQuestionStrategies: ["decision criterion", "constraint"], + acceptableQuestionStrategies: [ + "decision_threshold", + "evidence_gathering", + "definition", + ], notes: "The first question should establish whether more engineering capacity is justified before compensation or implementation details.", graph: makeScenarioGraph({ @@ -143,7 +147,11 @@ export const questionPriorityGeneralisationFixtures = [ "paint colour", "finance provider", ], - acceptableQuestionStrategies: ["decision criterion", "constraint"], + acceptableQuestionStrategies: [ + "decision_threshold", + "evidence_gathering", + "definition", + ], notes: "The first question should establish whether the fleet is failing a threshold that justifies replacement.", graph: makeScenarioGraph({ @@ -219,7 +227,7 @@ export const questionPriorityGeneralisationFixtures = [ "office location", "advertising channel", ], - acceptableQuestionStrategies: ["actor/customer", "decision criterion"], + acceptableQuestionStrategies: ["definition", "decision_threshold"], notes: "The first question should clarify the customer or value case for expansion before rollout logistics.", graph: makeScenarioGraph({ @@ -291,7 +299,7 @@ export const questionPriorityGeneralisationFixtures = [ "project-remaining-benefit", ], prohibitedFirstTopics: ["sunk cost", "project logo", "final launch date"], - acceptableQuestionStrategies: ["decision criterion", "objective"], + acceptableQuestionStrategies: ["decision_threshold", "definition"], notes: "The first question should establish remaining value or success threshold before sunk-cost framing or launch timing.", graph: makeScenarioGraph({ @@ -367,7 +375,11 @@ export const questionPriorityGeneralisationFixtures = [ "payment provider", "tier name", ], - acceptableQuestionStrategies: ["actor/customer", "decision criterion"], + acceptableQuestionStrategies: [ + "definition", + "decision_threshold", + "baseline_reconstruction", + ], notes: "The first question should establish who values paid support or what outcome would justify offering it before pricing details.", graph: makeScenarioGraph({ diff --git a/tests/graph/question-formulator.test.js b/tests/graph/question-formulator.test.js index eba2f8e..a0c6f64 100644 --- a/tests/graph/question-formulator.test.js +++ b/tests/graph/question-formulator.test.js @@ -1,5 +1,8 @@ import { describe, expect, it } from "vitest"; -import { formulateQuestion } from "@/lib/graph/question-formulator.js"; +import { + formulateQuestion, + selectInvestigationStrategy, +} from "@/lib/graph/question-formulator.js"; import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js"; function makeGraphFor(node, extra = {}) { @@ -14,7 +17,7 @@ function makeGraphFor(node, extra = {}) { } describe("formulateQuestion", () => { - it("commercial viability plus build decision produces a decision-criterion question", () => { + it("commercial viability plus build decision produces a decision-threshold question", () => { const unknown = makeNode({ id: "n-commercial", label: "Uncertainty regarding the commercial value of the product", @@ -42,7 +45,7 @@ describe("formulateQuestion", () => { const result = formulateQuestion({ node: unknown, graph }); - expect(result.strategy).toBe("decision criterion"); + expect(result.strategy).toBe("decision_threshold"); expect(result.question).toContain("What outcome"); expect(result.question.toLowerCase()).toContain("justify"); }); @@ -100,7 +103,7 @@ describe("formulateQuestion", () => { graph: makeGraphFor(unknown), }); - expect(result.strategy).toBe("evidence"); + expect(result.strategy).toBe("evidence_gathering"); expect(result.question).toContain("What evidence"); }); @@ -120,33 +123,41 @@ describe("formulateQuestion", () => { graph: makeGraphFor(unknown), }); - expect(result.strategy).toBe("baseline"); + expect(result.strategy).toBe("baseline_reconstruction"); expect(result.question).toContain("What was the comparable state before"); }); - it("unknown customer produces an actor/customer question", () => { + it("conflicting claim produces a contradiction-resolution question", () => { const unknown = makeNode({ - id: "n-customer", - label: "Target customer", + id: "n-conflict", + label: "Conflicting churn claim", description: - "Need to know the customer because value depends on who receives it.", + "Need to resolve the inconsistency because the current figures contradict each other.", kind: "unknown", status: "unknown", - confidence: "high", + confidence: "medium", + }); + + const contradiction = makeNode({ + id: "n-contradiction", + label: "Contradicted report", + description: "Two sources disagree about churn.", + kind: "conclusion", + status: "contradicted", + confidence: "low", + childIds: [unknown.id], }); const result = formulateQuestion({ node: unknown, - graph: makeGraphFor(unknown), + graph: makeGraphFor(unknown, { nodes: [contradiction] }), }); - expect(result.strategy).toBe("actor/customer"); - expect(result.question).toContain( - "Who experiences the problem or receives the value", - ); + expect(result.strategy).toBe("contradiction_resolution"); + expect(result.question).toContain("resolve the contradiction"); }); - it("constraint unknown produces a constraint question", () => { + it("constraint unknown uses evidence-gathering within the fixed strategy set", () => { const unknown = makeNode({ id: "n-constraint", label: "Budget constraint", @@ -162,8 +173,87 @@ describe("formulateQuestion", () => { graph: makeGraphFor(unknown), }); - expect(result.strategy).toBe("constraint"); - expect(result.question).toContain("What constraint most limits"); + expect(result.strategy).toBe("evidence_gathering"); + expect(result.question).toContain("What evidence"); + }); + + it("the same unknown can produce different questions when paired with different strategies", () => { + const unknown = makeNode({ + id: "n-same-unknown", + label: "Value threshold", + description: "Need to resolve the value threshold.", + kind: "unknown", + status: "unknown", + confidence: "high", + }); + + const decisionGraph = makeGraphFor(unknown, { + centralStatement: "We are deciding whether to launch this product.", + nodes: [ + makeNode({ + id: "n-decision", + label: "Launch decision", + description: "Decision depends on the value threshold.", + kind: "state", + status: "known", + confidence: "medium", + childIds: [unknown.id], + value: "Deciding whether to launch the product", + }), + ], + }); + + const definitionGraph = makeGraphFor(unknown, { + centralStatement: + "The team uses the term value threshold inconsistently.", + nodes: [ + makeNode({ + id: "n-definition", + label: "Definition disagreement", + description: + "Need a definition of value threshold before comparing options.", + kind: "state", + status: "known", + confidence: "medium", + childIds: [unknown.id], + }), + ], + }); + + const decisionResult = formulateQuestion({ + node: unknown, + graph: decisionGraph, + }); + const definitionResult = formulateQuestion({ + node: unknown, + graph: definitionGraph, + }); + + expect(decisionResult.strategy).toBe("decision_threshold"); + expect(definitionResult.strategy).toBe("definition"); + expect(decisionResult.question).not.toBe(definitionResult.question); + }); + + it("strategy selection is deterministic and explainable", () => { + const unknown = makeNode({ + id: "n-threshold", + label: "Success threshold", + description: + "Need the success threshold because the decision depends on it.", + kind: "unknown", + status: "unknown", + confidence: "high", + }); + const graph = makeGraphFor(unknown, { + centralStatement: "We need to decide whether to continue investing.", + }); + + const first = selectInvestigationStrategy({ node: unknown, graph }); + const second = selectInvestigationStrategy({ node: unknown, graph }); + + expect(first).toEqual(second); + expect(first.key).toBe("decision_threshold"); + expect(first.reason).toContain("threshold"); }); it("question is singular and answerable", () => { diff --git a/tests/graph/question-priority-generalisation.test.js b/tests/graph/question-priority-generalisation.test.js index 1a788a1..7eff090 100644 --- a/tests/graph/question-priority-generalisation.test.js +++ b/tests/graph/question-priority-generalisation.test.js @@ -127,27 +127,27 @@ describe("question priority generalisation", () => { { "nodeId": "hire-success-criteria", "scenario": "Should we hire another engineer?", - "strategy": "decision criterion", + "strategy": "decision_threshold", }, { "nodeId": "van-reliability-threshold", "scenario": "Should we replace the delivery vans?", - "strategy": "decision criterion", + "strategy": "decision_threshold", }, { "nodeId": "country-value-threshold", "scenario": "Should we launch in another country?", - "strategy": "actor/customer", + "strategy": "decision_threshold", }, { "nodeId": "project-benefit-threshold", "scenario": "Should we continue a project that is over budget?", - "strategy": "decision criterion", + "strategy": "decision_threshold", }, { "nodeId": "support-value-threshold", "scenario": "Should we introduce a paid support tier?", - "strategy": "actor/customer", + "strategy": "baseline_reconstruction", }, ] `); From 586802950d174653dd8c3d66ab0665a6d823e9c9 Mon Sep 17 00:00:00 2001 From: robbond Date: Sun, 2 Aug 2026 15:27:00 +0100 Subject: [PATCH 02/17] feat: explain deterministic unknown selection --- lib/graph/orchestrator.js | 29 +++- lib/graph/utils.js | 251 ++++++++++++++++++++++++++++--- tests/graph/orchestrator.test.js | 66 ++++++++ 3 files changed, 321 insertions(+), 25 deletions(-) diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index cbbc894..2f561ca 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -17,6 +17,7 @@ import { applyValidatedProposal } from "./apply-proposal.js"; import { buildGraphUpdatePrompt } from "./prompt-builder.js"; import { parseGraphUpdateProposal } from "./update-proposal.js"; import { + explainUnknownSelection, selectActiveUnknownCandidate, validateGraphReferences, } from "./utils.js"; @@ -31,7 +32,12 @@ function toValidationErrors(error) { ); } -function buildDiagnostics({ analysis, graph, graphReferenceValidation }) { +function buildDiagnostics({ + analysis, + graph, + graphReferenceValidation, + unknownSelectionExplanation, +}) { return { promptVersion: analysis?.promptVersion ?? null, modelName: analysis?.modelName ?? null, @@ -43,6 +49,7 @@ function buildDiagnostics({ analysis, graph, graphReferenceValidation }) { compatibilityApplied: analysis?.compatibilityApplied ?? false, compatibilityChanges: analysis?.compatibilityChanges ?? [], compatibilityWarnings: analysis?.compatibilityWarnings ?? [], + unknownSelectionExplanation: unknownSelectionExplanation ?? null, }; } @@ -54,6 +61,7 @@ function buildUpdateDiagnostics({ graph, graphReferenceValidation, selectedQuestion, + unknownSelectionExplanation, }) { return { promptVersion: promptVersion ?? "v0.4", @@ -71,6 +79,7 @@ function buildUpdateDiagnostics({ selectedQuestion?.investigationStrategy ?? selectedQuestion?.strategy ?? null, + unknownSelectionExplanation: unknownSelectionExplanation ?? null, }; } @@ -131,6 +140,10 @@ export async function startCase(body) { situationGraphSchema.parse(situationGraph); const graphReferenceValidation = validateGraphReferences(situationGraph); + const unknownSelectionExplanation = explainUnknownSelection( + situationGraph, + [], + ); if (!graphReferenceValidation.valid) { return { success: false, @@ -139,6 +152,7 @@ export async function startCase(body) { analysis, graph: situationGraph, graphReferenceValidation, + unknownSelectionExplanation, }), validationErrors: graphReferenceValidation.errors, statusCode: 500, @@ -153,6 +167,7 @@ export async function startCase(body) { analysis, graph: situationGraph, graphReferenceValidation, + unknownSelectionExplanation, }), }; } @@ -290,6 +305,10 @@ async function updateCaseWithDependencies(body, dependencies = {}) { graph: situationGraph, graphReferenceValidation: graphReferenceValidation, selectedQuestion: null, + unknownSelectionExplanation: explainUnknownSelection( + situationGraph, + situationGraph.resolvedNodeIds || [], + ), }), }, statusCode: @@ -320,6 +339,10 @@ async function updateCaseWithDependencies(body, dependencies = {}) { graph: applicationResult.updatedSituationGraph, graphReferenceValidation: applicationResult.graphReferenceValidation, selectedQuestion: applicationResult.selectedQuestion, + unknownSelectionExplanation: explainUnknownSelection( + applicationResult.updatedSituationGraph, + applicationResult.updatedSituationGraph.resolvedNodeIds || [], + ), }), }; } @@ -336,6 +359,10 @@ async function updateCaseWithDependencies(body, dependencies = {}) { graph: situationGraph, graphReferenceValidation, selectedQuestion: null, + unknownSelectionExplanation: explainUnknownSelection( + situationGraph, + situationGraph.resolvedNodeIds || [], + ), }), }; } diff --git a/lib/graph/utils.js b/lib/graph/utils.js index c1fc6cc..cd2cebf 100644 --- a/lib/graph/utils.js +++ b/lib/graph/utils.js @@ -95,6 +95,127 @@ function classifyUnknownPriority(text) { return matches; } +function buildScoreContributions( + matches, + downstreamCount, + unresolvedParentUnknownCount, +) { + const contributions = [ + { + rule: "downstream_dependencies", + value: downstreamCount, + weight: 4, + delta: downstreamCount * 4, + }, + ]; + + if (matches.objective) { + contributions.push({ + rule: "objective_match", + value: true, + weight: 12, + delta: 12, + }); + } + if (matches.actor) { + contributions.push({ + rule: "actor_match", + value: true, + weight: 10, + delta: 10, + }); + } + if (matches.criteria) { + contributions.push({ + rule: "criteria_match", + value: true, + weight: 11, + delta: 11, + }); + } + if (matches.measure) { + contributions.push({ + rule: "measure_match", + value: true, + weight: 8, + delta: 8, + }); + } + if (matches.terminology) { + contributions.push({ + rule: "terminology_match", + value: true, + weight: 7, + delta: 7, + }); + } + if (matches.constraint) { + contributions.push({ + rule: "constraint_match", + value: true, + weight: 9, + delta: 9, + }); + } + if (matches.pricing) { + contributions.push({ + rule: "pricing_penalty", + value: true, + weight: -8, + delta: -8, + }); + } + if (matches.implementation) { + contributions.push({ + rule: "implementation_penalty", + value: true, + weight: -10, + delta: -10, + }); + } + if (matches.optimisation) { + contributions.push({ + rule: "optimisation_penalty", + value: true, + weight: -9, + delta: -9, + }); + } + if (matches.speculative) { + contributions.push({ + rule: "speculative_penalty", + value: true, + weight: -12, + delta: -12, + }); + } + + if ( + matches.pricing && + !matches.objective && + !matches.criteria && + !matches.actor + ) { + contributions.push({ + rule: "isolated_pricing_penalty", + value: true, + weight: -6, + delta: -6, + }); + } + + if (unresolvedParentUnknownCount > 0) { + contributions.push({ + rule: "unresolved_prerequisite_penalty", + value: unresolvedParentUnknownCount, + weight: -7, + delta: unresolvedParentUnknownCount * -7, + }); + } + + return contributions; +} + export function scoreUnknownCandidate(graph, node, resolvedNodeIds = []) { const text = collectNodeText(node); const matches = classifyUnknownPriority(text); @@ -105,30 +226,15 @@ export function scoreUnknownCandidate(graph, node, resolvedNodeIds = []) { resolvedNodeIds, ); - let score = downstreamCount * 4; - - if (matches.objective) score += 12; - if (matches.actor) score += 10; - if (matches.criteria) score += 11; - if (matches.measure) score += 8; - if (matches.terminology) score += 7; - if (matches.constraint) score += 9; - - if (matches.pricing) score -= 8; - if (matches.implementation) score -= 10; - if (matches.optimisation) score -= 9; - if (matches.speculative) score -= 12; - - if ( - matches.pricing && - !matches.objective && - !matches.criteria && - !matches.actor - ) { - score -= 6; - } - - score -= unresolvedParentUnknownCount * 7; + const contributions = buildScoreContributions( + matches, + downstreamCount, + unresolvedParentUnknownCount, + ); + const score = contributions.reduce( + (total, contribution) => total + contribution.delta, + 0, + ); return { nodeId: node.id, @@ -137,6 +243,7 @@ export function scoreUnknownCandidate(graph, node, resolvedNodeIds = []) { downstreamCount, unresolvedParentUnknownCount, matches, + contributions, }; } @@ -406,6 +513,102 @@ export function selectActiveUnknownCandidate(graph, resolvedNodeIds) { }; } +export function explainUnknownSelection(graph, resolvedNodeIds = []) { + const unresolved = graph.nodes.filter( + (n) => n.kind === "unknown" && !resolvedNodeIds.includes(n.id), + ); + + if (unresolved.length === 0) { + return { + selectedNodeId: null, + selectedNodeLabel: null, + resolvedNodeIds: [...resolvedNodeIds], + candidates: [], + competitors: [], + tieBreakOrder: [ + "score_desc", + "downstreamCount_desc", + "unresolvedParentUnknownCount_asc", + "label_asc", + ], + summary: { + candidateCount: 0, + }, + }; + } + + const candidates = unresolved.map((node) => ({ + nodeId: node.id, + label: node.label, + ...scoreUnknownCandidate(graph, node, resolvedNodeIds), + })); + + candidates.sort((a, b) => { + if (b.score !== a.score) return b.score - a.score; + if (b.downstreamCount !== a.downstreamCount) { + return b.downstreamCount - a.downstreamCount; + } + if (a.unresolvedParentUnknownCount !== b.unresolvedParentUnknownCount) { + return a.unresolvedParentUnknownCount - b.unresolvedParentUnknownCount; + } + return a.label.localeCompare(b.label); + }); + + const selected = candidates[0]; + const competitors = candidates.slice(1).map((candidate) => ({ + nodeId: candidate.nodeId, + label: candidate.label, + score: candidate.score, + downstreamCount: candidate.downstreamCount, + unresolvedParentUnknownCount: candidate.unresolvedParentUnknownCount, + matches: candidate.matches, + contributions: candidate.contributions, + outrankedBy: { + scoreDelta: selected.score - candidate.score, + downstreamDelta: selected.downstreamCount - candidate.downstreamCount, + unresolvedPrerequisiteDelta: + candidate.unresolvedParentUnknownCount - + selected.unresolvedParentUnknownCount, + labelOrderWinner: + selected.score === candidate.score && + selected.downstreamCount === candidate.downstreamCount && + selected.unresolvedParentUnknownCount === + candidate.unresolvedParentUnknownCount + ? selected.label.localeCompare(candidate.label) <= 0 + ? selected.label + : candidate.label + : null, + }, + })); + + return { + selectedNodeId: selected.nodeId, + selectedNodeLabel: selected.label, + resolvedNodeIds: [...resolvedNodeIds], + tieBreakOrder: [ + "score_desc", + "downstreamCount_desc", + "unresolvedParentUnknownCount_asc", + "label_asc", + ], + candidates, + selected: { + nodeId: selected.nodeId, + label: selected.label, + score: selected.score, + downstreamCount: selected.downstreamCount, + unresolvedParentUnknownCount: selected.unresolvedParentUnknownCount, + matches: selected.matches, + contributions: selected.contributions, + }, + competitors, + summary: { + candidateCount: candidates.length, + selectedReason: `highest_score=${selected.score}; downstream=${selected.downstreamCount}; unresolved_prerequisites=${selected.unresolvedParentUnknownCount}`, + }, + }; +} + // ── Apply a graph update deterministically ── export function applyGraphUpdate(graph, update) { diff --git a/tests/graph/orchestrator.test.js b/tests/graph/orchestrator.test.js index 3a61642..2c241fa 100644 --- a/tests/graph/orchestrator.test.js +++ b/tests/graph/orchestrator.test.js @@ -178,6 +178,9 @@ describe("lib/graph/orchestrator startCase", () => { expect(result.success).toBe(true); expect(result.situationGraph.activeUnknownNodeId).toBeTruthy(); + expect(result.diagnostics.unknownSelectionExplanation?.selectedNodeId).toBe( + result.situationGraph.activeUnknownNodeId, + ); }); it("returns structured failure when graph reference validation fails", async () => { @@ -262,6 +265,69 @@ describe("lib/graph/orchestrator startCase", () => { expect(result.diagnostics.compatibilityChanges).toHaveLength(1); }); + it("preserves selected question bytes while adding selection explanation diagnostics", async () => { + const { updateCase } = await import("@/lib/graph/orchestrator.js"); + const provider = { + generateReconstruction: vi.fn().mockResolvedValue( + makeProposal({ + addedNodes: [ + makeNode({ + id: "n-build-decision", + label: "Build Confidence Engine decision", + description: "Decision introduced by the answer.", + kind: "state", + status: "supported", + confidence: "medium", + }), + makeNode({ + id: "n-commercial-value", + label: "Commercial value definition", + description: + "Need a concrete definition because the decision depends on it.", + kind: "unknown", + status: "unknown", + confidence: "high", + }), + ], + addedEdges: [ + { + id: "e-build-commercial-value", + fromNodeId: "n-build-decision", + toNodeId: "n-commercial-value", + relationship: "depends_on", + confidence: "medium", + description: + "The decision depends on commercial value definition.", + }, + ], + selectedQuestion: { + nodeId: "n-commercial-value", + question: + "How should commercial value be defined for this decision?", + reason: "Consequential unresolved uncertainty remains.", + }, + }), + ), + }; + + const first = await updateCase(makeUpdateRequest(), { + provider, + config: MOCK_CONFIG, + applyProposal: true, + }); + const second = await updateCase(makeUpdateRequest(), { + provider, + config: MOCK_CONFIG, + applyProposal: true, + }); + + expect(first.selectedQuestion.question).toBe( + second.selectedQuestion.question, + ); + expect(first.selectedQuestion.reason).toBe(second.selectedQuestion.reason); + expect(first.diagnostics.unknownSelectionExplanation).toBeTruthy(); + }); + it("produces a validated update proposal for a valid request", async () => { const { updateCase } = await import("@/lib/graph/orchestrator.js"); const provider = { From a1f6d0c2b9127ae94cd1df3cce448e87bcf8d75c Mon Sep 17 00:00:00 2001 From: robbond Date: Sun, 2 Aug 2026 15:40:06 +0100 Subject: [PATCH 03/17] test: inspect structural influence in unknown selection --- docs/v0.6-selection-influence-experiment.md | 50 ++++ .../selection-influence-diagnostic.test.js | 265 ++++++++++++++++++ 2 files changed, 315 insertions(+) create mode 100644 docs/v0.6-selection-influence-experiment.md create mode 100644 tests/graph/selection-influence-diagnostic.test.js diff --git a/docs/v0.6-selection-influence-experiment.md b/docs/v0.6-selection-influence-experiment.md new file mode 100644 index 0000000..6d1dfbf --- /dev/null +++ b/docs/v0.6-selection-influence-experiment.md @@ -0,0 +1,50 @@ +# v0.6 Selection Influence Experiment + +## Hypothesis + +The initial unknown selected for the revenue-versus-cash scenario may be driven more by graph structure, more by semantic keyword matches, or by both together. + +## Scenario + +`Revenue increased by 18%, but cash in the bank fell over the same period.` + +## Actual selected node + +- Node ID: `nqdzobz` +- Label: `Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).` +- Deterministic investigation strategy: `definition` +- Deterministic question: `What evidence would resolve whether magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts). is true?` + +## Structural contribution + +- Downstream dependency count: `0` +- Prerequisite position: no unresolved prerequisites; count `0` +- Dependency ordering / centrality: no candidate had downstream dependants or dependency depth advantage in the live graph + +## Semantic contribution + +- Objective: false +- Actor: false +- Criteria: false +- Measurement: false +- Terminology: false +- Constraint: false +- Pricing: false +- Implementation: false +- Optimisation: false +- Speculative: false +- Contribution list: only `downstream_dependencies` was present, with delta `0` + +## Counterfactual results + +- Live-shaped ordering: `nqdzobz` ranked above `niewza`, but both had score `0`, downstream `0`, and unresolved prerequisites `0` +- Links removed: ordering stayed the same, because the live graph already provided no differentiating structure between the two unknowns +- Wording neutralised: ordering flipped to the first unknown by neutral label order (`Unknown A` before `Unknown B`), showing the outcome remained tie-break-driven rather than structure-driven + +## Conclusion + +For this scenario, the actual winner was not selected because of graph structure and not selected because of semantic keyword weights. The live diagnostics show a complete tie on score, downstream influence, and prerequisite position, with every semantic match category false for both candidates. The winner was therefore chosen by the final tie-break rule, `label_asc`. + +## Is a scoring change justified? + +Not from this single experiment alone. The result shows a diagnostic gap for this scenario, but this task does not justify a scoring change by itself, and no scoring change is made. diff --git a/tests/graph/selection-influence-diagnostic.test.js b/tests/graph/selection-influence-diagnostic.test.js new file mode 100644 index 0000000..a9ac645 --- /dev/null +++ b/tests/graph/selection-influence-diagnostic.test.js @@ -0,0 +1,265 @@ +import { describe, expect, it } from "vitest"; +import { formulateQuestion } from "@/lib/graph/question-formulator.js"; +import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js"; +import { explainUnknownSelection } from "@/lib/graph/utils.js"; + +function buildLiveShapedGraph() { + const summary = makeNode({ + id: "nnvog0y", + label: + "Revenue grew by 18% while corporate cash reserves declined over an identical time frame.", + description: "Summary of the situation from the scenario text", + kind: "state", + status: "provisional", + confidence: "medium", + }); + const revenueObservation = makeNode({ + id: "nri36w9", + label: "Revenue increased by 18%.", + description: "Revenue increased by 18%.", + kind: "observation", + status: "supported", + confidence: "high", + evidenceIds: ["obs_rev"], + }); + const cashObservation = makeNode({ + id: "nnfc48j", + label: "Cash in the bank decreased over the same period.", + description: "Cash in the bank decreased over the same period.", + kind: "observation", + status: "supported", + confidence: "high", + evidenceIds: ["obs_cash"], + }); + const revenueMetric = makeNode({ + id: "nhsd6d5", + label: "Revenue metric (typically accrual-based income statement figure)", + description: + "Revenue metric (typically accrual-based income statement figure)", + kind: "metric", + status: "known", + confidence: "high", + }); + const cashMetric = makeNode({ + id: "neh5m6m", + label: + "Cash balance (liquidity measure on the balance sheet or cash flow statement)", + description: + "Cash balance (liquidity measure on the balance sheet or cash flow statement)", + kind: "metric", + status: "known", + confidence: "high", + }); + const directionalRelationship = makeNode({ + id: "nwo6070", + label: + "Divergent directional movement between top-line revenue growth and net cash position contraction.", + description: + "Divergent directional movement between top-line revenue growth and net cash position contraction.", + kind: "relationship", + status: "supported", + confidence: "high", + }); + const contradictionRelationship = makeNode({ + id: "nuiab02", + label: + "Apparent contradiction between profitability/revenue expansion and liquidity reduction.", + description: + "Apparent contradiction between profitability/revenue expansion and liquidity reduction.", + kind: "relationship", + status: "supported", + confidence: "medium", + }); + const cashTiming = makeNode({ + id: "niewza", + label: + "Whether revenue recognition timing differs from cash collection timing.", + description: + "Whether revenue recognition timing differs from cash collection timing.", + kind: "unknown", + status: "unknown", + confidence: "high", + }); + const cashOutflows = makeNode({ + id: "nqdzobz", + label: + "Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).", + description: + "Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).", + kind: "unknown", + status: "unknown", + confidence: "high", + }); + + const edges = [ + makeEdge({ + id: "e-revenue-summary", + fromNodeId: revenueObservation.id, + toNodeId: summary.id, + relationship: "supports", + description: "Revenue increase supports the scenario summary.", + }), + makeEdge({ + id: "e-cash-summary", + fromNodeId: cashObservation.id, + toNodeId: summary.id, + relationship: "supports", + description: "Cash decline supports the scenario summary.", + }), + makeEdge({ + id: "e-unk-niewza", + fromNodeId: cashTiming.id, + toNodeId: summary.id, + relationship: "depends_on", + description: + "Whether revenue recognition timing differs from cash collection timing. is an unresolved factor for this situation", + }), + makeEdge({ + id: "e-unk-nqdzobz", + fromNodeId: cashOutflows.id, + toNodeId: summary.id, + relationship: "depends_on", + description: + "Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts). is an unresolved factor for this situation", + }), + ]; + + return makeGraph({ + centralStatement: + "Revenue increased by 18%, but cash in the bank fell over the same period.", + nodes: [ + summary, + revenueObservation, + cashObservation, + revenueMetric, + cashMetric, + directionalRelationship, + contradictionRelationship, + cashTiming, + cashOutflows, + ], + edges, + activeUnknownNodeId: null, + resolvedNodeIds: [], + currentSummary: "Diagnostic selection influence fixture", + }); +} + +function orderCandidates(explanation) { + return explanation.candidates.map((candidate) => ({ + nodeId: candidate.nodeId, + label: candidate.label, + score: candidate.score, + downstreamCount: candidate.downstreamCount, + unresolvedParentUnknownCount: candidate.unresolvedParentUnknownCount, + })); +} + +function removeDependencyLinks(graph) { + const nodes = graph.nodes.map((node) => ({ + ...node, + dependsOn: [], + affects: [], + parentId: null, + childIds: [], + })); + const edges = (graph.edges || []).filter( + (edge) => edge.relationship !== "depends_on", + ); + return makeGraph({ ...graph, nodes, edges, activeUnknownNodeId: null }); +} + +function neutraliseUnknownWording(graph) { + let counter = 0; + const nodes = graph.nodes.map((node) => { + if (node.kind !== "unknown") return { ...node }; + counter += 1; + return { + ...node, + label: `Unknown ${String.fromCharCode(64 + counter)}`, + description: `Unknown factor ${counter} relevant to the scenario.`, + }; + }); + return makeGraph({ ...graph, nodes, activeUnknownNodeId: null }); +} + +describe("selection influence diagnostic", () => { + it("records ordering changes for live-shaped, structure-only, and wording-neutralised fixtures", () => { + const liveGraph = buildLiveShapedGraph(); + const liveExplanation = explainUnknownSelection(liveGraph, []); + const liveWinner = liveGraph.nodes.find( + (node) => node.id === liveExplanation.selectedNodeId, + ); + const liveQuestion = formulateQuestion({ + node: liveWinner, + graph: liveGraph, + }); + + const noLinksExplanation = explainUnknownSelection( + removeDependencyLinks(liveGraph), + [], + ); + const neutralWordingExplanation = explainUnknownSelection( + neutraliseUnknownWording(liveGraph), + [], + ); + + const diagnosticRecord = { + liveShapedCandidateOrdering: orderCandidates(liveExplanation), + noLinksCandidateOrdering: orderCandidates(noLinksExplanation), + neutralWordingCandidateOrdering: orderCandidates( + neutralWordingExplanation, + ), + selectedExplanationContributions: + liveExplanation.selected?.contributions ?? [], + selectedInvestigationStrategy: liveQuestion.strategy, + }; + + expect(diagnosticRecord.liveShapedCandidateOrdering).toEqual([ + { + nodeId: "nqdzobz", + label: + "Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).", + score: 0, + downstreamCount: 0, + unresolvedParentUnknownCount: 0, + }, + { + nodeId: "niewza", + label: + "Whether revenue recognition timing differs from cash collection timing.", + score: 0, + downstreamCount: 0, + unresolvedParentUnknownCount: 0, + }, + ]); + expect(diagnosticRecord.noLinksCandidateOrdering).toEqual( + diagnosticRecord.liveShapedCandidateOrdering, + ); + expect(diagnosticRecord.neutralWordingCandidateOrdering).toEqual([ + { + nodeId: "niewza", + label: "Unknown A", + score: 0, + downstreamCount: 0, + unresolvedParentUnknownCount: 0, + }, + { + nodeId: "nqdzobz", + label: "Unknown B", + score: 0, + downstreamCount: 0, + unresolvedParentUnknownCount: 0, + }, + ]); + expect(diagnosticRecord.selectedExplanationContributions).toEqual([ + { + rule: "downstream_dependencies", + value: 0, + weight: 4, + delta: 0, + }, + ]); + expect(diagnosticRecord.selectedInvestigationStrategy).toBe("definition"); + }); +}); From 51ce35621837c52660e6ecce02a581fd91ef7876 Mon Sep 17 00:00:00 2001 From: robbond Date: Sun, 2 Aug 2026 16:09:00 +0100 Subject: [PATCH 04/17] fix: handle unjustified unknown selection ties --- lib/graph/apply-proposal.js | 52 ++-- lib/graph/orchestrator.js | 57 +++- lib/graph/question-formulator.js | 155 +++++++++-- lib/graph/utils.js | 262 +++++++++++++----- tests/graph/orchestrator.test.js | 81 ++++++ tests/graph/question-formulator.test.js | 120 +++++++- .../selection-influence-diagnostic.test.js | 82 ++++-- tests/graph/utils.test.js | 67 +++++ 8 files changed, 731 insertions(+), 145 deletions(-) diff --git a/lib/graph/apply-proposal.js b/lib/graph/apply-proposal.js index ff82ce3..aabfdfc 100644 --- a/lib/graph/apply-proposal.js +++ b/lib/graph/apply-proposal.js @@ -1,5 +1,8 @@ import { describeGraph } from "./builder.js"; -import { formulateQuestion } from "./question-formulator.js"; +import { + formulateQuestion, + formulateTieResolutionQuestion, +} from "./question-formulator.js"; import { graphUpdateSchema, situationGraphSchema } from "./schema.js"; import { applyGraphUpdate, @@ -623,18 +626,25 @@ export function applyValidatedProposal({ situationGraph, proposal }) { updatedSituationGraph.resolvedNodeIds, ); - if (deterministicSelection?.nodeId) { + if ( + deterministicSelection?.status === "selected" && + deterministicSelection?.nodeId + ) { newActiveUnknownNodeId = deterministicSelection.nodeId; + } else if (deterministicSelection?.status === "ambiguous") { + newActiveUnknownNodeId = null; } updatedSituationGraph.activeUnknownNodeId = newActiveUnknownNodeId; updatedSituationGraph.currentSummary = describeGraph(updatedSituationGraph); - const selectedNode = deterministicSelection?.nodeId - ? updatedSituationGraph.nodes.find( - (node) => node.id === deterministicSelection.nodeId, - ) - : null; + const selectedNode = + deterministicSelection?.status === "selected" && + deterministicSelection?.nodeId + ? updatedSituationGraph.nodes.find( + (node) => node.id === deterministicSelection.nodeId, + ) + : null; const formulatedQuestion = selectedNode ? formulateQuestion({ node: selectedNode, @@ -645,20 +655,28 @@ export function applyValidatedProposal({ situationGraph, proposal }) { .filter( (value) => typeof value === "string" && value.trim().length > 0, ), + selectionState: deterministicSelection, }, }) : null; - const finalSelectedQuestion = deterministicSelection - ? { - nodeId: deterministicSelection.nodeId, - question: - formulatedQuestion?.question || deterministicSelection.question, - reason: formulatedQuestion?.reason || deterministicSelection.reason, - strategy: formulatedQuestion?.strategy, - investigationStrategy: formulatedQuestion?.investigationStrategy, - } - : null; + const finalSelectedQuestion = + deterministicSelection?.status === "ambiguous" + ? { + nodeId: null, + tiedCandidateIds: deterministicSelection.tiedCandidateIds, + ...formulateTieResolutionQuestion({ graph: updatedSituationGraph }), + } + : deterministicSelection?.status === "selected" + ? { + nodeId: deterministicSelection.nodeId, + question: + formulatedQuestion?.question || deterministicSelection.question, + reason: formulatedQuestion?.reason || deterministicSelection.reason, + strategy: formulatedQuestion?.strategy, + investigationStrategy: formulatedQuestion?.investigationStrategy, + } + : null; const resultGraphValidation = situationGraphSchema.safeParse( updatedSituationGraph, diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index 2f561ca..21d0c64 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -15,6 +15,7 @@ import { import { buildInitialGraph, describeGraph } from "./builder.js"; import { applyValidatedProposal } from "./apply-proposal.js"; import { buildGraphUpdatePrompt } from "./prompt-builder.js"; +import { formulateTieResolutionQuestion } from "./question-formulator.js"; import { parseGraphUpdateProposal } from "./update-proposal.js"; import { explainUnknownSelection, @@ -53,6 +54,26 @@ function buildDiagnostics({ }; } +function buildUnknownSelectionDiagnostics( + graph, + resolvedNodeIds = [], + selectedQuestion = null, +) { + const explanation = explainUnknownSelection(graph, resolvedNodeIds); + if (explanation.status === "ambiguous") { + return { + ...explanation, + tieResolutionQuestion: + selectedQuestion?.selectionStatus === "ambiguous" + ? selectedQuestion.question + : formulateTieResolutionQuestion({ graph }).question, + alphabeticalUsedAsReasoning: false, + }; + } + + return explanation; +} + function buildUpdateDiagnostics({ promptVersion, modelName, @@ -119,14 +140,17 @@ export async function startCase(body) { }); const currentSummary = describeGraph(initialGraph); + const deterministicSelection = selectActiveUnknownCandidate( + { + ...initialGraph, + resolvedNodeIds: [], + }, + [], + ); const activeUnknownNodeId = - selectActiveUnknownCandidate( - { - ...initialGraph, - resolvedNodeIds: [], - }, - [], - )?.nodeId ?? null; + deterministicSelection?.status === "selected" + ? deterministicSelection.nodeId + : null; const situationGraph = makeGraph({ centralStatement: scenario, @@ -140,9 +164,18 @@ export async function startCase(body) { situationGraphSchema.parse(situationGraph); const graphReferenceValidation = validateGraphReferences(situationGraph); - const unknownSelectionExplanation = explainUnknownSelection( + const selectedQuestion = + deterministicSelection?.status === "ambiguous" + ? { + id: "q_tie_resolution", + ...formulateTieResolutionQuestion({ graph: situationGraph }), + tiedCandidateIds: deterministicSelection.tiedCandidateIds, + } + : (analysis.nextQuestion ?? null); + const unknownSelectionExplanation = buildUnknownSelectionDiagnostics( situationGraph, [], + selectedQuestion, ); if (!graphReferenceValidation.valid) { return { @@ -162,7 +195,7 @@ export async function startCase(body) { return { success: true, situationGraph, - selectedQuestion: analysis.nextQuestion ?? null, + selectedQuestion, diagnostics: buildDiagnostics({ analysis, graph: situationGraph, @@ -339,9 +372,10 @@ async function updateCaseWithDependencies(body, dependencies = {}) { graph: applicationResult.updatedSituationGraph, graphReferenceValidation: applicationResult.graphReferenceValidation, selectedQuestion: applicationResult.selectedQuestion, - unknownSelectionExplanation: explainUnknownSelection( + unknownSelectionExplanation: buildUnknownSelectionDiagnostics( applicationResult.updatedSituationGraph, applicationResult.updatedSituationGraph.resolvedNodeIds || [], + applicationResult.selectedQuestion, ), }), }; @@ -359,9 +393,10 @@ async function updateCaseWithDependencies(body, dependencies = {}) { graph: situationGraph, graphReferenceValidation, selectedQuestion: null, - unknownSelectionExplanation: explainUnknownSelection( + unknownSelectionExplanation: buildUnknownSelectionDiagnostics( situationGraph, situationGraph.resolvedNodeIds || [], + null, ), }), }; diff --git a/lib/graph/question-formulator.js b/lib/graph/question-formulator.js index 60670b5..e9ebad8 100644 --- a/lib/graph/question-formulator.js +++ b/lib/graph/question-formulator.js @@ -11,6 +11,13 @@ function sentenceCase(value) { return trimmed.charAt(0).toLowerCase() + trimmed.slice(1); } +function stripTrailingPunctuation(value) { + return String(value || "") + .trim() + .replace(/[.?!:;]+$/g, "") + .trim(); +} + function buildNodeMap(graph) { return new Map((graph?.nodes || []).map((node) => [node.id, node])); } @@ -52,8 +59,8 @@ function collectResolvedContextValues(graph) { function extractMeaning(node) { const raw = `${node?.label || ""} ${node?.description || ""}`.trim(); - let meaning = String( - node?.label || node?.description || "this uncertainty", + let meaning = stripTrailingPunctuation( + String(node?.label || node?.description || "this uncertainty"), ).trim(); const lowered = normaliseText(raw); @@ -79,6 +86,84 @@ function extractMeaning(node) { return sentenceCase(meaning); } +function isDefinitionLikeUnknown(nodeText, text) { + return ( + /\b(define|definition|meaning|term|terminology)\b/.test(nodeText) || + (/\bdefinition\b/.test(text) && /\bdisagreement\b/.test(text)) || + (/\b(define|definition|meaning|term|terminology)\b/.test(text) && + /\b(unclear|ambiguous|inconsistent|undefined|used inconsistently)\b/.test( + text, + )) + ); +} + +function isClaimLikeUnknown(node, text) { + return ( + node?.kind === "reported_claim" || + node?.kind === "conclusion" || + /\b(claim|assertion|true|false|correct|incorrect|happened|happening)\b/.test( + text, + ) || + /^whether\b/i.test(String(node?.label || "").trim()) + ); +} + +function sanitizeQuestionText(question) { + return String(question || "") + .replace(/\)\.\s+/g, ") ") + .replace(/\s+/g, " ") + .trim(); +} + +function buildNeutralClarificationQuestion(meaning) { + return `What would clarify ${stripTrailingPunctuation(meaning)} in this situation?`; +} + +function buildEvidenceFallbackQuestion(meaning) { + return `What evidence would confirm or rule out ${stripTrailingPunctuation(meaning)}?`; +} + +function detectContradictionContext(graph) { + const central = stripTrailingPunctuation( + graph?.centralStatement || "this situation", + ); + const contradictionNode = (graph?.nodes || []).find((node) => { + const text = normaliseText(`${node.label} ${node.description}`); + return ( + node.kind === "relationship" && + /\b(contradiction|conflict|inconsistent|mismatch|divergent|opposing)\b/.test( + text, + ) + ); + }); + + return { + centralStatement: central, + contradictionLabel: stripTrailingPunctuation( + contradictionNode?.label || "", + ), + }; +} + +export function formulateTieResolutionQuestion({ graph }) { + const { centralStatement, contradictionLabel } = + detectContradictionContext(graph); + const focus = + centralStatement || contradictionLabel || "these conflicting signals"; + const question = sanitizeQuestionText( + `What changed during the period that could explain why ${focus}?`, + ); + + return { + question, + reason: + "Formulated to distinguish between tied unresolved explanations without prematurely choosing one branch.", + strategy: null, + investigationStrategy: null, + selectionStatus: "ambiguous", + }; +} + function extractActionPhrase(texts) { for (const text of texts) { const value = String(text || "").trim(); @@ -224,8 +309,7 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) { }); } - const hasDefinitionLanguage = - /\b(define|definition|meaning|term|terminology)\b/.test(text); + const hasDefinitionLanguage = isDefinitionLikeUnknown(nodeText, text); const hasPrimaryDefinitionLanguage = /\b(define|definition|meaning|term|terminology)\b/.test(nodeText); const hasCriteriaLanguage = @@ -243,9 +327,7 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) { ); const hasEvidenceLanguage = /\b(evidence|proof|validate|validation|signal|demand)\b/.test(text) || - node?.kind === "reported_claim" || - node?.kind === "conclusion" || - /\b(claim|assertion|true|false)\b/.test(text); + isClaimLikeUnknown(node, text); const hasContradictionLanguage = /\b(contradiction|contradict|conflict|inconsistent|inconsistency|disagree|mismatch)\b/.test( `${text} ${relatedText}`, @@ -323,16 +405,7 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) { }); } - return buildInvestigationStrategy({ - key: "definition", - reason: - "Selected as the deterministic fallback because clarifying the exact meaning of the unknown is the narrowest first step.", - node, - graph, - relatedNodes, - meaning, - actionPhrase, - }); + return null; } function buildQuestionFromStrategy(strategy) { @@ -344,7 +417,7 @@ function buildQuestionFromStrategy(strategy) { case "definition": return `What does ${strategy.meaning} mean in this situation?`; case "evidence_gathering": - return `What evidence would show whether ${strategy.meaning} is true?`; + return `What evidence would clarify ${stripTrailingPunctuation(strategy.meaning)}?`; case "baseline_reconstruction": return `What was the comparable state before ${strategy.meaning}?`; case "contradiction_resolution": @@ -382,6 +455,9 @@ function validateFormulatedQuestion(question, meaning) { if (/^how should uncertainty regarding\b/i.test(trimmed)) return false; if (/^what would resolve uncertainty regarding\b/i.test(trimmed)) return false; + if (/\)\.\s+[A-Z]/.test(trimmed)) return false; + if (/\bis true\?$/i.test(trimmed) && !/^whether\b/i.test(meaning)) + return false; if ( /\bprice|pricing|price point\b/i.test(trimmed) && !/\bprice\b/i.test(meaning) @@ -399,22 +475,55 @@ function validateFormulatedQuestion(question, meaning) { } export function formulateQuestion({ node, graph, context = {} }) { + if (context.selectionState?.status === "ambiguous") { + return formulateTieResolutionQuestion({ graph }); + } + const investigationStrategy = selectInvestigationStrategy({ node, graph, context, }); - let question = buildQuestionFromStrategy(investigationStrategy); + let question = investigationStrategy + ? buildQuestionFromStrategy(investigationStrategy) + : buildNeutralClarificationQuestion(extractMeaning(node)); - if (!validateFormulatedQuestion(question, investigationStrategy.meaning)) { - question = `What evidence would resolve whether ${investigationStrategy.meaning} is true?`; + question = sanitizeQuestionText(question); + + const fallbackMeaning = extractMeaning(node); + if ( + !validateFormulatedQuestion( + question, + investigationStrategy?.meaning || fallbackMeaning, + ) + ) { + question = sanitizeQuestionText( + investigationStrategy && + isClaimLikeUnknown( + node, + normaliseText( + collectRelatedNodes(node, graph) + .map( + (relatedNode) => + `${relatedNode.label} ${relatedNode.description}`, + ) + .concat([node?.label, node?.description]) + .filter(Boolean) + .join(" "), + ), + ) + ? buildEvidenceFallbackQuestion(fallbackMeaning) + : buildNeutralClarificationQuestion(fallbackMeaning), + ); } return { question, - reason: `Formulated from graph context using the ${investigationStrategy.key} investigation strategy.`, - strategy: investigationStrategy.key, + reason: investigationStrategy + ? `Formulated from graph context using the ${investigationStrategy.key} investigation strategy.` + : "Formulated as a neutral clarification question because no narrower investigation strategy clearly applied.", + strategy: investigationStrategy?.key ?? null, investigationStrategy, }; } diff --git a/lib/graph/utils.js b/lib/graph/utils.js index cd2cebf..134c24d 100644 --- a/lib/graph/utils.js +++ b/lib/graph/utils.js @@ -216,6 +216,119 @@ function buildScoreContributions( return contributions; } +function getMeaningfulSemanticContributions(contributions = []) { + return contributions + .filter( + (contribution) => + contribution.rule !== "downstream_dependencies" && + contribution.rule !== "unresolved_prerequisite_penalty" && + contribution.delta !== 0, + ) + .map((contribution) => ({ + rule: contribution.rule, + delta: contribution.delta, + })); +} + +function buildCandidateDisplayOrder(candidates) { + return [...candidates].sort((a, b) => { + if (b.score !== a.score) return b.score - a.score; + if (b.downstreamCount !== a.downstreamCount) { + return b.downstreamCount - a.downstreamCount; + } + if (a.unresolvedParentUnknownCount !== b.unresolvedParentUnknownCount) { + return a.unresolvedParentUnknownCount - b.unresolvedParentUnknownCount; + } + return a.label.localeCompare(b.label); + }); +} + +function semanticSignature(candidate) { + return JSON.stringify( + getMeaningfulSemanticContributions(candidate.contributions), + ); +} + +function classifyCandidateOrdering(candidates) { + const displayOrder = buildCandidateDisplayOrder(candidates); + const best = displayOrder[0] ?? null; + if (!best) { + return { + displayOrder, + best: null, + leadingCandidates: [], + status: "no_candidates", + tieType: "none", + usedAlphabeticalOrdering: false, + reason: "No unresolved unknown candidates remain.", + }; + } + + const topScoreCandidates = displayOrder.filter( + (candidate) => candidate.score === best.score, + ); + + if (topScoreCandidates.length === 1) { + return { + displayOrder, + best, + leadingCandidates: [best], + status: "selected", + tieType: "none", + usedAlphabeticalOrdering: false, + reason: `Clear winner by total score (${best.score}).`, + }; + } + + const topStructuralCandidates = topScoreCandidates.filter( + (candidate) => + candidate.downstreamCount === best.downstreamCount && + candidate.unresolvedParentUnknownCount === + best.unresolvedParentUnknownCount, + ); + + if (topStructuralCandidates.length === 1) { + return { + displayOrder, + best, + leadingCandidates: [best], + status: "selected", + tieType: "structural_tie", + usedAlphabeticalOrdering: false, + reason: + "Score tie was resolved by downstream dependency count or prerequisite ordering.", + }; + } + + const topSemanticSignature = semanticSignature(best); + const semanticPeers = topStructuralCandidates.filter( + (candidate) => semanticSignature(candidate) === topSemanticSignature, + ); + + if (semanticPeers.length !== topStructuralCandidates.length) { + return { + displayOrder, + best: null, + leadingCandidates: topStructuralCandidates, + status: "ambiguous", + tieType: "semantic_tie", + usedAlphabeticalOrdering: false, + reason: + "Leading candidates remain tied after score and structural checks, but differ in semantic contribution patterns.", + }; + } + + return { + displayOrder, + best: null, + leadingCandidates: topStructuralCandidates, + status: "ambiguous", + tieType: "complete_unresolved_tie", + usedAlphabeticalOrdering: false, + reason: "No justified distinction between leading unknowns.", + }; +} + export function scoreUnknownCandidate(graph, node, resolvedNodeIds = []) { const text = collectNodeText(node); const matches = classifyUnknownPriority(text); @@ -490,21 +603,36 @@ export function selectActiveUnknownCandidate(graph, resolvedNodeIds) { ...scoreUnknownCandidate(graph, node, resolvedNodeIds), })); - scoredCandidates.sort((a, b) => { - if (b.score !== a.score) return b.score - a.score; - if (b.downstreamCount !== a.downstreamCount) { - return b.downstreamCount - a.downstreamCount; - } - if (a.unresolvedParentUnknownCount !== b.unresolvedParentUnknownCount) { - return a.unresolvedParentUnknownCount - b.unresolvedParentUnknownCount; - } - return a.node.label.localeCompare(b.node.label); - }); + const selection = classifyCandidateOrdering( + scoredCandidates.map(({ node, ...candidate }) => ({ + ...candidate, + node, + })), + ); - const best = scoredCandidates[0]; + if (selection.status === "ambiguous") { + return { + selectedNode: null, + status: "ambiguous", + tieType: selection.tieType, + tiedCandidateIds: selection.leadingCandidates.map( + (candidate) => candidate.nodeId, + ), + displayOrder: selection.displayOrder.map((candidate) => candidate.nodeId), + reason: selection.reason, + }; + } + + const best = selection.best; if (!best) return null; return { + selectedNode: { + nodeId: best.node.id, + label: best.node.label, + }, + status: "selected", + tieType: selection.tieType, nodeId: best.node.id, label: best.node.label, score: best.score, @@ -522,7 +650,10 @@ export function explainUnknownSelection(graph, resolvedNodeIds = []) { return { selectedNodeId: null, selectedNodeLabel: null, + status: "no_candidates", + tieType: "none", resolvedNodeIds: [...resolvedNodeIds], + tiedCandidateIds: [], candidates: [], competitors: [], tieBreakOrder: [ @@ -543,68 +674,75 @@ export function explainUnknownSelection(graph, resolvedNodeIds = []) { ...scoreUnknownCandidate(graph, node, resolvedNodeIds), })); - candidates.sort((a, b) => { - if (b.score !== a.score) return b.score - a.score; - if (b.downstreamCount !== a.downstreamCount) { - return b.downstreamCount - a.downstreamCount; - } - if (a.unresolvedParentUnknownCount !== b.unresolvedParentUnknownCount) { - return a.unresolvedParentUnknownCount - b.unresolvedParentUnknownCount; - } - return a.label.localeCompare(b.label); - }); - - const selected = candidates[0]; - const competitors = candidates.slice(1).map((candidate) => ({ - nodeId: candidate.nodeId, - label: candidate.label, - score: candidate.score, - downstreamCount: candidate.downstreamCount, - unresolvedParentUnknownCount: candidate.unresolvedParentUnknownCount, - matches: candidate.matches, - contributions: candidate.contributions, - outrankedBy: { - scoreDelta: selected.score - candidate.score, - downstreamDelta: selected.downstreamCount - candidate.downstreamCount, - unresolvedPrerequisiteDelta: - candidate.unresolvedParentUnknownCount - - selected.unresolvedParentUnknownCount, - labelOrderWinner: - selected.score === candidate.score && - selected.downstreamCount === candidate.downstreamCount && - selected.unresolvedParentUnknownCount === - candidate.unresolvedParentUnknownCount - ? selected.label.localeCompare(candidate.label) <= 0 - ? selected.label - : candidate.label - : null, - }, - })); + const selection = classifyCandidateOrdering(candidates); + const orderedCandidates = selection.displayOrder; + const selected = selection.best; + const competitors = orderedCandidates + .filter((candidate) => candidate.nodeId !== selected?.nodeId) + .map((candidate) => ({ + nodeId: candidate.nodeId, + label: candidate.label, + score: candidate.score, + downstreamCount: candidate.downstreamCount, + unresolvedParentUnknownCount: candidate.unresolvedParentUnknownCount, + matches: candidate.matches, + contributions: candidate.contributions, + outrankedBy: { + scoreDelta: (selected?.score ?? candidate.score) - candidate.score, + downstreamDelta: + (selected?.downstreamCount ?? candidate.downstreamCount) - + candidate.downstreamCount, + unresolvedPrerequisiteDelta: + candidate.unresolvedParentUnknownCount - + (selected?.unresolvedParentUnknownCount ?? + candidate.unresolvedParentUnknownCount), + labelOrderWinner: + selected && + selected.score === candidate.score && + selected.downstreamCount === candidate.downstreamCount && + selected.unresolvedParentUnknownCount === + candidate.unresolvedParentUnknownCount + ? selected.label.localeCompare(candidate.label) <= 0 + ? selected.label + : candidate.label + : null, + }, + })); return { - selectedNodeId: selected.nodeId, - selectedNodeLabel: selected.label, + selectedNodeId: selected?.nodeId ?? null, + selectedNodeLabel: selected?.label ?? null, + status: selection.status, + tieType: selection.tieType, resolvedNodeIds: [...resolvedNodeIds], + tiedCandidateIds: selection.leadingCandidates.map( + (candidate) => candidate.nodeId, + ), tieBreakOrder: [ "score_desc", "downstreamCount_desc", "unresolvedParentUnknownCount_asc", "label_asc", ], - candidates, - selected: { - nodeId: selected.nodeId, - label: selected.label, - score: selected.score, - downstreamCount: selected.downstreamCount, - unresolvedParentUnknownCount: selected.unresolvedParentUnknownCount, - matches: selected.matches, - contributions: selected.contributions, - }, + alphabeticalUsedAsReasoning: false, + candidates: orderedCandidates, + selected: selected + ? { + nodeId: selected.nodeId, + label: selected.label, + score: selected.score, + downstreamCount: selected.downstreamCount, + unresolvedParentUnknownCount: selected.unresolvedParentUnknownCount, + matches: selected.matches, + contributions: selected.contributions, + } + : null, competitors, summary: { - candidateCount: candidates.length, - selectedReason: `highest_score=${selected.score}; downstream=${selected.downstreamCount}; unresolved_prerequisites=${selected.unresolvedParentUnknownCount}`, + candidateCount: orderedCandidates.length, + selectedReason: selected + ? `highest_score=${selected.score}; downstream=${selected.downstreamCount}; unresolved_prerequisites=${selected.unresolvedParentUnknownCount}` + : selection.reason, }, }; } diff --git a/tests/graph/orchestrator.test.js b/tests/graph/orchestrator.test.js index 2c241fa..e91458c 100644 --- a/tests/graph/orchestrator.test.js +++ b/tests/graph/orchestrator.test.js @@ -183,6 +183,87 @@ describe("lib/graph/orchestrator startCase", () => { ); }); + it("returns an ambiguous tie result instead of choosing by label order", async () => { + mockAnalyseScenario.mockResolvedValue( + makeAnalysisResult({ + reconstruction: { + summary: "Revenue up while cash falls", + actors: [], + systemsOrObjects: [], + expectedStates: [], + observedStates: [ + { + id: "obs-1", + label: "Revenue increased by 18%.", + description: "Revenue increased by 18%.", + confidence: "high", + }, + { + id: "obs-2", + label: "Cash in the bank decreased over the same period.", + description: "Cash in the bank decreased over the same period.", + confidence: "high", + }, + ], + differences: [], + knownTransitions: [], + unexplainedTransitions: [], + contradictions: [ + { + id: "c-1", + label: + "Apparent contradiction between profitability/revenue expansion and liquidity reduction.", + description: + "Apparent contradiction between profitability/revenue expansion and liquidity reduction.", + confidence: "medium", + }, + ], + importantUnknowns: [ + { + id: "unk-1", + label: + "Whether revenue recognition timing differs from cash collection timing.", + description: + "Whether revenue recognition timing differs from cash collection timing.", + confidence: "high", + }, + { + id: "unk-2", + label: + "Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).", + description: + "Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).", + confidence: "high", + }, + ], + plausibleInterpretations: [], + }, + }), + ); + const { startCase } = await import("@/lib/graph/orchestrator.js"); + + const result = await startCase({ + scenario: + "Revenue increased by 18%, but cash in the bank fell over the same period.", + }); + + expect(result.success).toBe(true); + expect(result.situationGraph.activeUnknownNodeId).toBeNull(); + expect(result.selectedQuestion).toMatchObject({ + id: "q_tie_resolution", + selectionStatus: "ambiguous", + question: + "What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?", + tiedCandidateIds: expect.arrayContaining([expect.any(String)]), + }); + expect(result.diagnostics.unknownSelectionExplanation).toMatchObject({ + status: "ambiguous", + tieType: "complete_unresolved_tie", + selectedNodeId: null, + alphabeticalUsedAsReasoning: false, + }); + }); + it("returns structured failure when graph reference validation fails", async () => { mockAnalyseScenario.mockResolvedValue(makeAnalysisResult()); const utils = await import("@/lib/graph/utils.js"); diff --git a/tests/graph/question-formulator.test.js b/tests/graph/question-formulator.test.js index a0c6f64..366d404 100644 --- a/tests/graph/question-formulator.test.js +++ b/tests/graph/question-formulator.test.js @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { formulateQuestion, + formulateTieResolutionQuestion, selectInvestigationStrategy, } from "@/lib/graph/question-formulator.js"; import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js"; @@ -178,16 +179,24 @@ describe("formulateQuestion", () => { }); it("the same unknown can produce different questions when paired with different strategies", () => { - const unknown = makeNode({ - id: "n-same-unknown", + const thresholdUnknown = makeNode({ + id: "n-threshold-unknown", label: "Value threshold", description: "Need to resolve the value threshold.", kind: "unknown", status: "unknown", confidence: "high", }); + const definitionUnknown = makeNode({ + id: "n-definition-unknown", + label: "Value term", + description: "Need to resolve what value term refers to in this context.", + kind: "unknown", + status: "unknown", + confidence: "high", + }); - const decisionGraph = makeGraphFor(unknown, { + const decisionGraph = makeGraphFor(thresholdUnknown, { centralStatement: "We are deciding whether to launch this product.", nodes: [ makeNode({ @@ -197,35 +206,35 @@ describe("formulateQuestion", () => { kind: "state", status: "known", confidence: "medium", - childIds: [unknown.id], + childIds: [thresholdUnknown.id], value: "Deciding whether to launch the product", }), ], }); - const definitionGraph = makeGraphFor(unknown, { + const definitionGraph = makeGraphFor(definitionUnknown, { centralStatement: "The team uses the term value threshold inconsistently.", nodes: [ makeNode({ id: "n-definition", - label: "Definition disagreement", + label: "Definition disagreement about value threshold", description: - "Need a definition of value threshold before comparing options.", + "Need a definition of value threshold because the term is used inconsistently before comparing options.", kind: "state", status: "known", confidence: "medium", - childIds: [unknown.id], + childIds: [definitionUnknown.id], }), ], }); const decisionResult = formulateQuestion({ - node: unknown, + node: thresholdUnknown, graph: decisionGraph, }); const definitionResult = formulateQuestion({ - node: unknown, + node: definitionUnknown, graph: definitionGraph, }); @@ -296,4 +305,95 @@ describe("formulateQuestion", () => { "What would resolve uncertainty regarding", ); }); + + it("ambiguous contradiction produces a broad distinguishing question without accounting jargon", () => { + const unknown = makeNode({ + id: "n-cause-a", + label: "Cash outflow cause", + description: "Unclear explanation for the contradiction.", + kind: "unknown", + status: "unknown", + confidence: "high", + }); + const contradiction = makeNode({ + id: "n-contradiction", + label: "Divergent movement between revenue and cash", + description: "Two signals moved in opposite directions.", + kind: "relationship", + status: "supported", + confidence: "medium", + }); + const graph = makeGraphFor(unknown, { + centralStatement: + "Revenue increased by 18%, but cash in the bank fell over the same period.", + nodes: [contradiction], + }); + + const result = formulateTieResolutionQuestion({ graph }); + + expect(result.question).toBe( + "What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?", + ); + expect(result.question.toLowerCase()).not.toMatch( + /accounts receivable|capex|debt repayments|working capital/, + ); + }); + + it("definition is selected only for genuine definition unknowns", () => { + const unknown = makeNode({ + id: "n-definition-only", + label: "Definition of success criteria", + description: "The term is used inconsistently and needs a definition.", + kind: "unknown", + status: "unknown", + confidence: "medium", + }); + + const result = formulateQuestion({ + node: unknown, + graph: makeGraphFor(unknown), + }); + expect(result.strategy).toBe("definition"); + }); + + it("an unknown about possible causes does not become a definition question", () => { + const unknown = makeNode({ + id: "n-causes", + label: "Possible causes of the divergence", + description: "Several causes may explain the divergence.", + kind: "unknown", + status: "unknown", + confidence: "medium", + }); + + const result = formulateQuestion({ + node: unknown, + graph: makeGraphFor(unknown), + }); + expect(result.strategy).toBeNull(); + expect(result.question).toBe( + "What would clarify possible causes of the divergence in this situation?", + ); + }); + + it("malformed punctuation is rejected", () => { + const unknown = makeNode({ + id: "n-punct", + label: "Magnitude and nature of cash outflows (operating expenses).", + description: + "Magnitude and nature of cash outflows (operating expenses).", + kind: "unknown", + status: "unknown", + confidence: "medium", + }); + + const result = formulateQuestion({ + node: unknown, + graph: makeGraphFor(unknown), + }); + expect(result.question).not.toContain("). is true?"); + expect(result.question).toBe( + "What would clarify magnitude and nature of cash outflows (operating expenses) in this situation?", + ); + }); }); diff --git a/tests/graph/selection-influence-diagnostic.test.js b/tests/graph/selection-influence-diagnostic.test.js index a9ac645..a38521b 100644 --- a/tests/graph/selection-influence-diagnostic.test.js +++ b/tests/graph/selection-influence-diagnostic.test.js @@ -1,7 +1,13 @@ import { describe, expect, it } from "vitest"; -import { formulateQuestion } from "@/lib/graph/question-formulator.js"; +import { + formulateQuestion, + formulateTieResolutionQuestion, +} from "@/lib/graph/question-formulator.js"; import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js"; -import { explainUnknownSelection } from "@/lib/graph/utils.js"; +import { + explainUnknownSelection, + selectActiveUnknownCandidate, +} from "@/lib/graph/utils.js"; function buildLiveShapedGraph() { const summary = makeNode({ @@ -184,37 +190,57 @@ function neutraliseUnknownWording(graph) { } describe("selection influence diagnostic", () => { - it("records ordering changes for live-shaped, structure-only, and wording-neutralised fixtures", () => { + it("records ambiguous ordering changes for live-shaped, structure-only, and wording-neutralised fixtures", () => { const liveGraph = buildLiveShapedGraph(); const liveExplanation = explainUnknownSelection(liveGraph, []); - const liveWinner = liveGraph.nodes.find( - (node) => node.id === liveExplanation.selectedNodeId, - ); - const liveQuestion = formulateQuestion({ - node: liveWinner, - graph: liveGraph, - }); + const liveSelection = selectActiveUnknownCandidate(liveGraph, []); + const tieQuestion = formulateTieResolutionQuestion({ graph: liveGraph }); const noLinksExplanation = explainUnknownSelection( removeDependencyLinks(liveGraph), [], ); + const noLinksSelection = selectActiveUnknownCandidate( + removeDependencyLinks(liveGraph), + [], + ); const neutralWordingExplanation = explainUnknownSelection( neutraliseUnknownWording(liveGraph), [], ); + const neutralSelection = selectActiveUnknownCandidate( + neutraliseUnknownWording(liveGraph), + [], + ); + + const fallbackQuestion = formulateQuestion({ + node: liveGraph.nodes.find((node) => node.id === "nqdzobz"), + graph: liveGraph, + }); const diagnosticRecord = { + liveStatus: liveExplanation.status, liveShapedCandidateOrdering: orderCandidates(liveExplanation), + liveTiedCandidateIds: liveExplanation.tiedCandidateIds, noLinksCandidateOrdering: orderCandidates(noLinksExplanation), + noLinksStatus: noLinksExplanation.status, neutralWordingCandidateOrdering: orderCandidates( neutralWordingExplanation, ), - selectedExplanationContributions: - liveExplanation.selected?.contributions ?? [], - selectedInvestigationStrategy: liveQuestion.strategy, + neutralStatus: neutralWordingExplanation.status, + selectedExplanationContributions: liveExplanation.selected?.contributions, + tieQuestion: tieQuestion.question, + liveSelection, + noLinksSelection, + neutralSelection, + fallbackQuestion, }; + expect(diagnosticRecord.liveStatus).toBe("ambiguous"); + expect(diagnosticRecord.liveTiedCandidateIds).toEqual([ + "nqdzobz", + "niewza", + ]); expect(diagnosticRecord.liveShapedCandidateOrdering).toEqual([ { nodeId: "nqdzobz", @@ -233,9 +259,17 @@ describe("selection influence diagnostic", () => { unresolvedParentUnknownCount: 0, }, ]); + expect(diagnosticRecord.liveSelection).toMatchObject({ + selectedNode: null, + status: "ambiguous", + tieType: "complete_unresolved_tie", + tiedCandidateIds: ["nqdzobz", "niewza"], + }); expect(diagnosticRecord.noLinksCandidateOrdering).toEqual( diagnosticRecord.liveShapedCandidateOrdering, ); + expect(diagnosticRecord.noLinksStatus).toBe("ambiguous"); + expect(diagnosticRecord.noLinksSelection.status).toBe("ambiguous"); expect(diagnosticRecord.neutralWordingCandidateOrdering).toEqual([ { nodeId: "niewza", @@ -252,14 +286,18 @@ describe("selection influence diagnostic", () => { unresolvedParentUnknownCount: 0, }, ]); - expect(diagnosticRecord.selectedExplanationContributions).toEqual([ - { - rule: "downstream_dependencies", - value: 0, - weight: 4, - delta: 0, - }, - ]); - expect(diagnosticRecord.selectedInvestigationStrategy).toBe("definition"); + expect(diagnosticRecord.neutralStatus).toBe("ambiguous"); + expect(diagnosticRecord.neutralSelection.status).toBe("ambiguous"); + expect(diagnosticRecord.selectedExplanationContributions).toBeUndefined(); + expect(diagnosticRecord.tieQuestion).toBe( + "What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?", + ); + expect(diagnosticRecord.tieQuestion.toLowerCase()).not.toMatch( + /accounts receivable|capex|debt repayments|working capital/, + ); + expect(diagnosticRecord.fallbackQuestion.strategy).toBeNull(); + expect(diagnosticRecord.fallbackQuestion.question).toBe( + "What would clarify magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts) in this situation?", + ); }); }); diff --git a/tests/graph/utils.test.js b/tests/graph/utils.test.js index 2e6a0b8..3de4b30 100644 --- a/tests/graph/utils.test.js +++ b/tests/graph/utils.test.js @@ -456,6 +456,7 @@ describe("selectActiveUnknownCandidate", () => { const result = selectActiveUnknownCandidate(graph, []); expect(result.nodeId).toBe("unknown-a"); // Has more dependents (score 2 vs 0) + expect(result.status).toBe("selected"); }); it("returns one candidate (not array)", () => { @@ -619,6 +620,72 @@ describe("selectActiveUnknownCandidate", () => { const childScore = scoreUnknownCandidate(graph, childUnknown, []); expect(parentScore.score).toBeGreaterThan(childScore.score); }); + + it("returns ambiguous for a complete unresolved tie instead of label-based winner", () => { + const unknownA = makeNode({ + id: "tie-a", + label: "Magnitude and nature of cash outflows", + description: "Magnitude and nature of cash outflows.", + kind: "unknown", + status: "unknown", + confidence: "high", + }); + const unknownB = makeNode({ + id: "tie-b", + label: + "Whether revenue recognition timing differs from cash collection timing", + description: + "Whether revenue recognition timing differs from cash collection timing.", + kind: "unknown", + status: "unknown", + confidence: "high", + }); + const graph = makeGraph({ + centralStatement: + "Revenue increased by 18%, but cash in the bank fell over the same period.", + nodes: [unknownA, unknownB], + edges: [], + activeUnknownNodeId: null, + resolvedNodeIds: [], + currentSummary: "Tie case", + }); + + const result = selectActiveUnknownCandidate(graph, []); + expect(result).toMatchObject({ + selectedNode: null, + status: "ambiguous", + tieType: "complete_unresolved_tie", + tiedCandidateIds: ["tie-a", "tie-b"], + }); + expect(result.nodeId).toBeUndefined(); + }); + + it("alphabetical renaming does not resolve a complete tie", () => { + const unknownA = makeNode({ + id: "tie-a", + label: "Unknown B", + description: "Unknown factor one.", + kind: "unknown", + }); + const unknownB = makeNode({ + id: "tie-b", + label: "Unknown A", + description: "Unknown factor two.", + kind: "unknown", + }); + const graph = makeGraph({ + centralStatement: "Two conflicting signals remain unresolved.", + nodes: [unknownA, unknownB], + edges: [], + activeUnknownNodeId: null, + resolvedNodeIds: [], + currentSummary: "Tie case", + }); + + const result = selectActiveUnknownCandidate(graph, []); + expect(result.status).toBe("ambiguous"); + expect(result.tiedCandidateIds.sort()).toEqual(["tie-a", "tie-b"]); + }); }); describe("applyGraphUpdate", () => { From b84989b96a07d5ea29cad847e812a69419ab5254 Mon Sep 17 00:00:00 2001 From: robbond Date: Sun, 2 Aug 2026 16:17:37 +0100 Subject: [PATCH 05/17] test: verify ambiguity handling across domains --- docs/v0.6-ambiguity-generalisation.md | 40 ++++ tests/fixtures/ambiguity-generalisation.js | 198 +++++++++++++++++++ tests/graph/ambiguity-generalisation.test.js | 132 +++++++++++++ 3 files changed, 370 insertions(+) create mode 100644 docs/v0.6-ambiguity-generalisation.md create mode 100644 tests/fixtures/ambiguity-generalisation.js create mode 100644 tests/graph/ambiguity-generalisation.test.js diff --git a/docs/v0.6-ambiguity-generalisation.md b/docs/v0.6-ambiguity-generalisation.md new file mode 100644 index 0000000..6ae698e --- /dev/null +++ b/docs/v0.6-ambiguity-generalisation.md @@ -0,0 +1,40 @@ +# v0.6 Ambiguity Generalisation + +## Hypothesis + +If the selector truly handles unjustified contradiction ties generically, it should return ambiguity across multiple domains without preferring one explanation by wording alone. + +## Scenarios + +1. Revenue increased by 18%, but cash in the bank fell over the same period. +2. Customer satisfaction scores increased, but complaints also increased. +3. Average delivery time decreased by 25%, but order cancellations increased. +4. Website traffic doubled, but sales remained unchanged. +5. Production output increased by 30%, but quality defects also increased. + +## Observed behaviour + +All five fixtures produced the same pattern: + +- candidate count: 2 +- selector status: `ambiguous` +- tie reason: `No justified distinction between leading unknowns.` +- no explanation was favoured +- one broad investigation question was produced from the central contradiction +- neutral label renaming did not collapse ambiguity into a winner + +## Repeated failure patterns + +None observed across two or more scenarios. + +The current ambiguity handling generalised cleanly across the five contradiction fixtures. + +## Corrections + +No production correction was required in this task. + +## Lessons learned + +- The current ambiguity path appears domain-agnostic when structure and semantic weights remain intentionally non-discriminating. +- Central-statement-based tie questions are broad enough to avoid prematurely backing one branch. +- The most useful regression signal is whether ambiguity survives neutral relabelling, not whether one label sorts ahead of another in display order. diff --git a/tests/fixtures/ambiguity-generalisation.js b/tests/fixtures/ambiguity-generalisation.js new file mode 100644 index 0000000..af3ce14 --- /dev/null +++ b/tests/fixtures/ambiguity-generalisation.js @@ -0,0 +1,198 @@ +import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js"; + +function buildAmbiguityFixture({ + key, + scenario, + summaryLabel, + contradictionLabel, + observationLabels, + unknownLabels, + disallowedQuestionTerms, +}) { + const summary = makeNode({ + id: `${key}-summary`, + label: summaryLabel, + description: "Summary of the situation from the scenario text", + kind: "state", + status: "provisional", + confidence: "medium", + }); + + const contradiction = makeNode({ + id: `${key}-contradiction`, + label: contradictionLabel, + description: contradictionLabel, + kind: "relationship", + status: "supported", + confidence: "medium", + }); + + const observations = observationLabels.map((label, index) => + makeNode({ + id: `${key}-obs-${index + 1}`, + label, + description: label, + kind: "observation", + status: "supported", + confidence: "high", + }), + ); + + const unknowns = unknownLabels.map((label, index) => + makeNode({ + id: `${key}-unknown-${index + 1}`, + label, + description: label, + kind: "unknown", + status: "unknown", + confidence: "high", + }), + ); + + const edges = [ + ...observations.map((node) => + makeEdge({ + id: `${node.id}-supports-summary`, + fromNodeId: node.id, + toNodeId: summary.id, + relationship: "supports", + description: `${node.label} supports the summary.`, + }), + ), + ...unknowns.map((node) => + makeEdge({ + id: `${node.id}-depends-summary`, + fromNodeId: node.id, + toNodeId: summary.id, + relationship: "depends_on", + description: `${node.label} is an unresolved factor for this situation.`, + }), + ), + ]; + + return { + key, + scenario, + disallowedQuestionTerms, + graph: makeGraph({ + centralStatement: scenario, + nodes: [summary, contradiction, ...observations, ...unknowns], + edges, + activeUnknownNodeId: null, + resolvedNodeIds: [], + currentSummary: `Ambiguity fixture for ${key}`, + }), + }; +} + +export const ambiguityGeneralisationFixtures = [ + buildAmbiguityFixture({ + key: "revenue-cash", + scenario: + "Revenue increased by 18%, but cash in the bank fell over the same period.", + summaryLabel: "Revenue rose while cash fell", + contradictionLabel: + "Contradiction between revenue improvement and lower cash reserves.", + observationLabels: [ + "Revenue increased by 18%.", + "Cash in the bank decreased over the same period.", + ], + unknownLabels: [ + "Possible explanation for the contradiction from one side of the situation.", + "Possible explanation for the contradiction from another side of the situation.", + ], + disallowedQuestionTerms: [ + "accounts receivable", + "capex", + "debt repayments", + "working capital", + ], + }), + buildAmbiguityFixture({ + key: "satisfaction-complaints", + scenario: + "Customer satisfaction scores increased, but complaints also increased.", + summaryLabel: "Satisfaction scores rose while complaints also rose", + contradictionLabel: + "Contradiction between higher satisfaction scores and higher complaint volume.", + observationLabels: [ + "Customer satisfaction scores increased.", + "Complaints increased.", + ], + unknownLabels: [ + "Possible explanation for why the positive signal and negative signal moved together.", + "Another possible explanation for why the positive signal and negative signal moved together.", + ], + disallowedQuestionTerms: [ + "net promoter", + "ticket backlog", + "call deflection", + "support queue", + ], + }), + buildAmbiguityFixture({ + key: "delivery-cancellations", + scenario: + "Average delivery time decreased by 25%, but order cancellations increased.", + summaryLabel: "Delivery became faster while cancellations increased", + contradictionLabel: + "Contradiction between faster delivery and more order cancellations.", + observationLabels: [ + "Average delivery time decreased by 25%.", + "Order cancellations increased.", + ], + unknownLabels: [ + "Possible explanation for why the faster result did not reduce the negative result.", + "Another possible explanation for why the faster result did not reduce the negative result.", + ], + disallowedQuestionTerms: [ + "fulfilment", + "last mile", + "warehouse", + "routing", + ], + }), + buildAmbiguityFixture({ + key: "traffic-sales", + scenario: "Website traffic doubled, but sales remained unchanged.", + summaryLabel: "Website traffic doubled while sales stayed flat", + contradictionLabel: + "Contradiction between much higher traffic and unchanged sales.", + observationLabels: [ + "Website traffic doubled.", + "Sales remained unchanged.", + ], + unknownLabels: [ + "Possible explanation for why the stronger signal did not change the outcome.", + "Another possible explanation for why the stronger signal did not change the outcome.", + ], + disallowedQuestionTerms: [ + "conversion funnel", + "campaign attribution", + "landing page", + "checkout flow", + ], + }), + buildAmbiguityFixture({ + key: "output-defects", + scenario: + "Production output increased by 30%, but quality defects also increased.", + summaryLabel: "Production output rose while defects also rose", + contradictionLabel: + "Contradiction between higher output and more quality defects.", + observationLabels: [ + "Production output increased by 30%.", + "Quality defects increased.", + ], + unknownLabels: [ + "Possible explanation for why the gain came with a worsening result.", + "Another possible explanation for why the gain came with a worsening result.", + ], + disallowedQuestionTerms: [ + "scrap rate", + "throughput", + "yield", + "root cause", + ], + }), +]; diff --git a/tests/graph/ambiguity-generalisation.test.js b/tests/graph/ambiguity-generalisation.test.js new file mode 100644 index 0000000..d31f46b --- /dev/null +++ b/tests/graph/ambiguity-generalisation.test.js @@ -0,0 +1,132 @@ +import { describe, expect, it } from "vitest"; +import { + formulateQuestion, + formulateTieResolutionQuestion, +} from "@/lib/graph/question-formulator.js"; +import { + explainUnknownSelection, + selectActiveUnknownCandidate, +} from "@/lib/graph/utils.js"; +import { ambiguityGeneralisationFixtures } from "@/tests/fixtures/ambiguity-generalisation.js"; + +function neutraliseUnknownLabels(graph) { + let counter = 0; + return { + ...graph, + nodes: graph.nodes.map((node) => { + if (node.kind !== "unknown") return { ...node }; + counter += 1; + return { + ...node, + label: `Unknown ${String.fromCharCode(64 + counter)}`, + description: `Unknown factor ${counter}.`, + }; + }), + }; +} + +function isSingleQuestion(question) { + return (question.match(/\?/g) || []).length === 1; +} + +describe("ambiguity generalisation", () => { + it("preserves ambiguity across contradiction scenarios without favouring one explanation", () => { + const summary = ambiguityGeneralisationFixtures.map((fixture) => { + const explanation = explainUnknownSelection(fixture.graph, []); + const selection = selectActiveUnknownCandidate(fixture.graph, []); + const neutralExplanation = explainUnknownSelection( + neutraliseUnknownLabels(fixture.graph), + [], + ); + const tieQuestion = formulateTieResolutionQuestion({ + graph: fixture.graph, + }); + const representativeUnknown = fixture.graph.nodes.find( + (node) => node.kind === "unknown", + ); + const fallbackQuestion = formulateQuestion({ + node: representativeUnknown, + graph: fixture.graph, + }); + + const lowerQuestion = tieQuestion.question.toLowerCase(); + for (const term of fixture.disallowedQuestionTerms) { + expect(lowerQuestion).not.toContain(term.toLowerCase()); + } + + expect(explanation.status).toBe("ambiguous"); + expect(selection.status).toBe("ambiguous"); + expect(selection.selectedNode).toBeNull(); + expect(explanation.selectedNodeId).toBeNull(); + expect(explanation.candidates).toHaveLength(2); + expect(explanation.summary.selectedReason).toBe( + "No justified distinction between leading unknowns.", + ); + expect(explanation.alphabeticalUsedAsReasoning).toBe(false); + expect(neutralExplanation.status).toBe("ambiguous"); + expect(isSingleQuestion(tieQuestion.question)).toBe(true); + expect(tieQuestion.question.toLowerCase()).not.toContain(" and "); + expect(tieQuestion.question.toLowerCase()).not.toContain(" or "); + + return { + scenario: fixture.scenario, + candidateCount: explanation.candidates.length, + ambiguityStatus: explanation.status, + tieReason: explanation.summary.selectedReason, + investigationStrategy: tieQuestion.strategy, + question: tieQuestion.question, + explanationFavoured: explanation.selectedNodeId !== null, + }; + }); + + expect(summary).toMatchInlineSnapshot(` + [ + { + "ambiguityStatus": "ambiguous", + "candidateCount": 2, + "explanationFavoured": false, + "investigationStrategy": null, + "question": "What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?", + "scenario": "Revenue increased by 18%, but cash in the bank fell over the same period.", + "tieReason": "No justified distinction between leading unknowns.", + }, + { + "ambiguityStatus": "ambiguous", + "candidateCount": 2, + "explanationFavoured": false, + "investigationStrategy": null, + "question": "What changed during the period that could explain why Customer satisfaction scores increased, but complaints also increased?", + "scenario": "Customer satisfaction scores increased, but complaints also increased.", + "tieReason": "No justified distinction between leading unknowns.", + }, + { + "ambiguityStatus": "ambiguous", + "candidateCount": 2, + "explanationFavoured": false, + "investigationStrategy": null, + "question": "What changed during the period that could explain why Average delivery time decreased by 25%, but order cancellations increased?", + "scenario": "Average delivery time decreased by 25%, but order cancellations increased.", + "tieReason": "No justified distinction between leading unknowns.", + }, + { + "ambiguityStatus": "ambiguous", + "candidateCount": 2, + "explanationFavoured": false, + "investigationStrategy": null, + "question": "What changed during the period that could explain why Website traffic doubled, but sales remained unchanged?", + "scenario": "Website traffic doubled, but sales remained unchanged.", + "tieReason": "No justified distinction between leading unknowns.", + }, + { + "ambiguityStatus": "ambiguous", + "candidateCount": 2, + "explanationFavoured": false, + "investigationStrategy": null, + "question": "What changed during the period that could explain why Production output increased by 30%, but quality defects also increased?", + "scenario": "Production output increased by 30%, but quality defects also increased.", + "tieReason": "No justified distinction between leading unknowns.", + }, + ] + `); + }); +}); From 0c7558d31fa31c733c7ad45d8ed08f8ea655e24f Mon Sep 17 00:00:00 2001 From: robbond Date: Sun, 2 Aug 2026 16:28:11 +0100 Subject: [PATCH 06/17] feat: introduce comparability assessment before contradiction reasoning --- docs/v0.6-comparability-experiment.md | 27 +++ lib/graph/question-formulator.js | 118 +++++++++++++ tests/fixtures/comparability-assessment.js | 164 +++++++++++++++++++ tests/graph/comparability-assessment.test.js | 85 ++++++++++ tests/graph/orchestrator.test.js | 6 +- 5 files changed, 399 insertions(+), 1 deletion(-) create mode 100644 docs/v0.6-comparability-experiment.md create mode 100644 tests/fixtures/comparability-assessment.js create mode 100644 tests/graph/comparability-assessment.test.js diff --git a/docs/v0.6-comparability-experiment.md b/docs/v0.6-comparability-experiment.md new file mode 100644 index 0000000..15d1d17 --- /dev/null +++ b/docs/v0.6-comparability-experiment.md @@ -0,0 +1,27 @@ +# v0.6 Comparability Experiment + +## Hypothesis + +The engine should confirm that observations are comparable before treating their difference as a contradiction that needs explanatory follow-up. + +## Fixtures + +1. Revenue increased by 18%, but cash in the bank fell over the same period. +2. Complaints increased. Production increased. +3. Average delivery time decreased by 25%, but order cancellations increased. +4. Customer satisfaction increased, but complaints increased. +5. Temperature increased. Ice melted. +6. Sales doubled. Sales doubled. + +## Results + +- The first four scenarios repeated the same failure pattern: contradiction-level investigation could begin before comparability was established. +- A deterministic comparability gate corrected that by producing one comparison question first. +- Temperature increased / Ice melted was treated as comparability confirmed, so no comparison question was asked. +- Sales doubled / Sales doubled was treated as comparability confirmed, and contradiction reasoning was not needed. + +## Whether comparability should become a permanent reasoning stage + +Yes, in minimal deterministic form. + +The repeated pattern appeared in four scenarios, so a small pre-contradiction comparability assessment is justified. diff --git a/lib/graph/question-formulator.js b/lib/graph/question-formulator.js index e9ebad8..13a813e 100644 --- a/lib/graph/question-formulator.js +++ b/lib/graph/question-formulator.js @@ -123,6 +123,105 @@ function buildEvidenceFallbackQuestion(meaning) { return `What evidence would confirm or rule out ${stripTrailingPunctuation(meaning)}?`; } +function collectObservationNodes(graph) { + return (graph?.nodes || []).filter( + (node) => node.kind === "observation" && node.status === "supported", + ); +} + +function analyseObservationText(text) { + const normalised = normaliseText(text); + return { + text, + normalised, + isMeasurementLike: + /\b(increase|increased|decrease|decreased|fell|rose|doubled|halved|remained|average|score|scores|rate|time|traffic|sales|output|defects|complaints|production|revenue|cash|temperature|quality)\b/.test( + normalised, + ) || /%|percent/.test(String(text || "")), + timeframeMentioned: + /\b(period|timeframe|quarter|month|week|year|day|annual|daily|weekly|monthly|same period)\b/.test( + normalised, + ), + scaleMentioned: /\b(average|rate|score|scores|per|percent|%)\b/.test( + normalised, + ), + unitMentioned: + /\b(celsius|fahrenheit|minutes|minute|hours|hour|days|day|units|sales|traffic|cash|revenue|complaints|defects)\b/.test( + normalised, + ), + }; +} + +export function assessComparability(graph) { + const observations = collectObservationNodes(graph); + const centralText = normaliseText(graph?.centralStatement || ""); + const profiles = observations.map((node) => + analyseObservationText(`${node.label} ${node.description}`), + ); + + if (profiles.length < 2) { + return { + comparabilityStatus: "confirmed", + reason: "Fewer than two supported observations need comparison.", + contradictionReasoningAllowed: true, + }; + } + + if ( + profiles.every((profile) => profile.normalised === profiles[0].normalised) + ) { + return { + comparabilityStatus: "confirmed", + reason: "The observations restate the same measurement.", + contradictionReasoningAllowed: false, + }; + } + + if (profiles.some((profile) => !profile.isMeasurementLike)) { + return { + comparabilityStatus: "confirmed", + reason: "The observations are not competing like-for-like measurements.", + contradictionReasoningAllowed: true, + }; + } + + const hasExplicitTimeframe = + /\b(period|timeframe|quarter|month|week|year|day|same period)\b/.test( + centralText, + ) || profiles.every((profile) => profile.timeframeMentioned); + + const hasSharedScale = profiles.every((profile) => profile.scaleMentioned); + const hasSharedUnits = profiles.every((profile) => profile.unitMentioned); + + if (!hasExplicitTimeframe || !hasSharedScale || !hasSharedUnits) { + return { + comparabilityStatus: "uncertain", + reason: + "Comparability between the observations is not yet established across period, scale, or measurement basis.", + contradictionReasoningAllowed: false, + }; + } + + return { + comparabilityStatus: "uncertain", + reason: + "The observations appear comparable in form, but the basis for comparing them is still not established.", + contradictionReasoningAllowed: false, + }; +} + +function buildComparabilityQuestion(graph, assessment) { + const centralText = normaliseText(graph?.centralStatement || ""); + const mentionsPeriod = + /\b(period|timeframe|quarter|month|week|year|day)\b/.test(centralText); + + if (mentionsPeriod) { + return "Were these figures measured on the same basis and at the same scale?"; + } + + return "Were these figures measured over the same period and at the same scale?"; +} + function detectContradictionContext(graph) { const central = stripTrailingPunctuation( graph?.centralStatement || "this situation", @@ -146,6 +245,22 @@ function detectContradictionContext(graph) { } export function formulateTieResolutionQuestion({ graph }) { + const comparability = assessComparability(graph); + if (comparability.comparabilityStatus === "uncertain") { + return { + question: buildComparabilityQuestion(graph, comparability), + reason: + "Formulated to confirm whether the observations are comparable before exploring competing explanations.", + strategy: null, + investigationStrategy: null, + selectionStatus: "ambiguous", + comparabilityStatus: comparability.comparabilityStatus, + comparabilityReason: comparability.reason, + contradictionReasoningAllowed: + comparability.contradictionReasoningAllowed, + }; + } + const { centralStatement, contradictionLabel } = detectContradictionContext(graph); const focus = @@ -161,6 +276,9 @@ export function formulateTieResolutionQuestion({ graph }) { strategy: null, investigationStrategy: null, selectionStatus: "ambiguous", + comparabilityStatus: comparability.comparabilityStatus, + comparabilityReason: comparability.reason, + contradictionReasoningAllowed: comparability.contradictionReasoningAllowed, }; } diff --git a/tests/fixtures/comparability-assessment.js b/tests/fixtures/comparability-assessment.js new file mode 100644 index 0000000..db64b6b --- /dev/null +++ b/tests/fixtures/comparability-assessment.js @@ -0,0 +1,164 @@ +import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js"; + +function buildComparabilityFixture({ + key, + scenario, + observationLabels, + contradictionLabel, + expectedComparabilityStatus, + expectsComparisonQuestion, +}) { + const summary = makeNode({ + id: `${key}-summary`, + label: scenario, + description: "Summary of the situation from the scenario text", + kind: "state", + status: "provisional", + confidence: "medium", + }); + + const observations = observationLabels.map((label, index) => + makeNode({ + id: `${key}-obs-${index + 1}`, + label, + description: label, + kind: "observation", + status: "supported", + confidence: "high", + }), + ); + + const contradiction = contradictionLabel + ? [ + makeNode({ + id: `${key}-contradiction`, + label: contradictionLabel, + description: contradictionLabel, + kind: "relationship", + status: "supported", + confidence: "medium", + }), + ] + : []; + + const unknowns = [ + makeNode({ + id: `${key}-unknown-a`, + label: "Possible explanation from one side of the situation.", + description: "Possible explanation from one side of the situation.", + kind: "unknown", + status: "unknown", + confidence: "high", + }), + makeNode({ + id: `${key}-unknown-b`, + label: "Possible explanation from another side of the situation.", + description: "Possible explanation from another side of the situation.", + kind: "unknown", + status: "unknown", + confidence: "high", + }), + ]; + + const edges = [ + ...observations.map((node) => + makeEdge({ + id: `${node.id}-supports-summary`, + fromNodeId: node.id, + toNodeId: summary.id, + relationship: "supports", + description: `${node.label} supports the summary.`, + }), + ), + ...unknowns.map((node) => + makeEdge({ + id: `${node.id}-depends-summary`, + fromNodeId: node.id, + toNodeId: summary.id, + relationship: "depends_on", + description: `${node.label} is an unresolved factor for this situation.`, + }), + ), + ]; + + return { + key, + scenario, + expectedComparabilityStatus, + expectsComparisonQuestion, + graph: makeGraph({ + centralStatement: scenario, + nodes: [summary, ...observations, ...contradiction, ...unknowns], + edges, + activeUnknownNodeId: null, + resolvedNodeIds: [], + currentSummary: `Comparability fixture for ${key}`, + }), + }; +} + +export const comparabilityAssessmentFixtures = [ + buildComparabilityFixture({ + key: "revenue-cash", + scenario: + "Revenue increased by 18%, but cash in the bank fell over the same period.", + observationLabels: [ + "Revenue increased by 18%.", + "Cash in the bank decreased over the same period.", + ], + contradictionLabel: + "Contradiction between revenue improvement and lower cash reserves.", + expectedComparabilityStatus: "uncertain", + expectsComparisonQuestion: true, + }), + buildComparabilityFixture({ + key: "complaints-production", + scenario: "Complaints increased. Production increased.", + observationLabels: ["Complaints increased.", "Production increased."], + contradictionLabel: + "Possible contradiction between complaints and production movement.", + expectedComparabilityStatus: "uncertain", + expectsComparisonQuestion: true, + }), + buildComparabilityFixture({ + key: "delivery-cancellations", + scenario: + "Average delivery time decreased by 25%, but order cancellations increased.", + observationLabels: [ + "Average delivery time decreased by 25%.", + "Order cancellations increased.", + ], + contradictionLabel: + "Contradiction between faster delivery and more cancellations.", + expectedComparabilityStatus: "uncertain", + expectsComparisonQuestion: true, + }), + buildComparabilityFixture({ + key: "satisfaction-complaints", + scenario: "Customer satisfaction increased, but complaints increased.", + observationLabels: [ + "Customer satisfaction increased.", + "Complaints increased.", + ], + contradictionLabel: + "Contradiction between satisfaction improvement and more complaints.", + expectedComparabilityStatus: "uncertain", + expectsComparisonQuestion: true, + }), + buildComparabilityFixture({ + key: "temperature-ice", + scenario: "Temperature increased. Ice melted.", + observationLabels: ["Temperature increased.", "Ice melted."], + contradictionLabel: null, + expectedComparabilityStatus: "confirmed", + expectsComparisonQuestion: false, + }), + buildComparabilityFixture({ + key: "sales-same", + scenario: "Sales doubled. Sales doubled.", + observationLabels: ["Sales doubled.", "Sales doubled."], + contradictionLabel: null, + expectedComparabilityStatus: "confirmed", + expectsComparisonQuestion: false, + }), +]; diff --git a/tests/graph/comparability-assessment.test.js b/tests/graph/comparability-assessment.test.js new file mode 100644 index 0000000..0d38918 --- /dev/null +++ b/tests/graph/comparability-assessment.test.js @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; +import { + assessComparability, + formulateTieResolutionQuestion, +} from "@/lib/graph/question-formulator.js"; +import { explainUnknownSelection } from "@/lib/graph/utils.js"; +import { comparabilityAssessmentFixtures } from "@/tests/fixtures/comparability-assessment.js"; + +describe("comparability assessment", () => { + it("generates comparison questions only when comparability is uncertain", () => { + const summary = comparabilityAssessmentFixtures.map((fixture) => { + const assessment = assessComparability(fixture.graph); + const question = formulateTieResolutionQuestion({ graph: fixture.graph }); + const ambiguity = explainUnknownSelection(fixture.graph, []); + + expect(assessment.comparabilityStatus).toBe( + fixture.expectedComparabilityStatus, + ); + expect(question.comparabilityStatus).toBe( + fixture.expectedComparabilityStatus, + ); + + if (fixture.expectsComparisonQuestion) { + expect(question.question.toLowerCase()).toContain("same"); + expect(question.contradictionReasoningAllowed).toBe(false); + } else { + expect(question.question.toLowerCase()).not.toContain( + "same period and at the same scale", + ); + } + + if (fixture.key !== "sales-same") { + expect(ambiguity.status).toBe("ambiguous"); + } + + return { + scenario: fixture.scenario, + comparabilityStatus: assessment.comparabilityStatus, + contradictionReasoningAllowed: assessment.contradictionReasoningAllowed, + question: question.question, + }; + }); + + expect(summary).toMatchInlineSnapshot(` + [ + { + "comparabilityStatus": "uncertain", + "contradictionReasoningAllowed": false, + "question": "Were these figures measured on the same basis and at the same scale?", + "scenario": "Revenue increased by 18%, but cash in the bank fell over the same period.", + }, + { + "comparabilityStatus": "uncertain", + "contradictionReasoningAllowed": false, + "question": "Were these figures measured over the same period and at the same scale?", + "scenario": "Complaints increased. Production increased.", + }, + { + "comparabilityStatus": "uncertain", + "contradictionReasoningAllowed": false, + "question": "Were these figures measured over the same period and at the same scale?", + "scenario": "Average delivery time decreased by 25%, but order cancellations increased.", + }, + { + "comparabilityStatus": "uncertain", + "contradictionReasoningAllowed": false, + "question": "Were these figures measured over the same period and at the same scale?", + "scenario": "Customer satisfaction increased, but complaints increased.", + }, + { + "comparabilityStatus": "confirmed", + "contradictionReasoningAllowed": true, + "question": "What changed during the period that could explain why Temperature increased. Ice melted?", + "scenario": "Temperature increased. Ice melted.", + }, + { + "comparabilityStatus": "confirmed", + "contradictionReasoningAllowed": false, + "question": "What changed during the period that could explain why Sales doubled. Sales doubled?", + "scenario": "Sales doubled. Sales doubled.", + }, + ] + `); + }); +}); diff --git a/tests/graph/orchestrator.test.js b/tests/graph/orchestrator.test.js index e91458c..3693a34 100644 --- a/tests/graph/orchestrator.test.js +++ b/tests/graph/orchestrator.test.js @@ -253,14 +253,18 @@ describe("lib/graph/orchestrator startCase", () => { id: "q_tie_resolution", selectionStatus: "ambiguous", question: - "What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?", + "Were these figures measured on the same basis and at the same scale?", tiedCandidateIds: expect.arrayContaining([expect.any(String)]), + comparabilityStatus: "uncertain", + contradictionReasoningAllowed: false, }); expect(result.diagnostics.unknownSelectionExplanation).toMatchObject({ status: "ambiguous", tieType: "complete_unresolved_tie", selectedNodeId: null, alphabeticalUsedAsReasoning: false, + tieResolutionQuestion: + "Were these figures measured on the same basis and at the same scale?", }); }); From c97f5f7303a80509b13a1b7f0532a3606314a15f Mon Sep 17 00:00:00 2001 From: robbond Date: Sun, 2 Aug 2026 16:40:24 +0100 Subject: [PATCH 07/17] feat: classify observation relationships after comparability --- docs/v0.6-comparability-experiment.md | 15 +- lib/graph/question-formulator.js | 178 ++++++++++++++++++- tests/graph/ambiguity-generalisation.test.js | 11 +- tests/graph/comparability-assessment.test.js | 141 +++++++++++---- tests/graph/question-formulator.test.js | 21 ++- 5 files changed, 316 insertions(+), 50 deletions(-) diff --git a/docs/v0.6-comparability-experiment.md b/docs/v0.6-comparability-experiment.md index 15d1d17..627cd53 100644 --- a/docs/v0.6-comparability-experiment.md +++ b/docs/v0.6-comparability-experiment.md @@ -17,8 +17,19 @@ The engine should confirm that observations are comparable before treating their - The first four scenarios repeated the same failure pattern: contradiction-level investigation could begin before comparability was established. - A deterministic comparability gate corrected that by producing one comparison question first. -- Temperature increased / Ice melted was treated as comparability confirmed, so no comparison question was asked. -- Sales doubled / Sales doubled was treated as comparability confirmed, and contradiction reasoning was not needed. +- Confirmed comparability did not by itself imply contradiction. +- Temperature increased / Ice melted was reclassified as a compatible relationship, so no contradiction question was asked. +- Sales doubled / Sales doubled was reclassified as duplicate observations, so no follow-up question was asked. + +## Relationship classification stage + +After comparability assessment, observations now pass through a deterministic relationship classification stage: + +- `contradictory` +- `compatible` +- `potentially_related` +- `duplicate` +- `insufficient_information` ## Whether comparability should become a permanent reasoning stage diff --git a/lib/graph/question-formulator.js b/lib/graph/question-formulator.js index 13a813e..3e60238 100644 --- a/lib/graph/question-formulator.js +++ b/lib/graph/question-formulator.js @@ -210,6 +210,139 @@ export function assessComparability(graph) { }; } +function extractObservationConcepts(profile) { + const concepts = new Set(); + const text = profile.normalised; + const conceptPatterns = [ + ["sales", /\bsales\b/], + ["revenue", /\brevenue\b/], + ["cash", /\bcash\b/], + ["complaints", /\bcomplaints?\b/], + ["production", /\bproduction\b/], + ["delivery_time", /\bdelivery time\b|\baverage delivery time\b/], + ["cancellations", /\bcancellations?\b/], + ["satisfaction", /\bsatisfaction\b/], + ["temperature", /\btemperature\b/], + ["ice", /\bice\b/], + ["traffic", /\btraffic\b/], + ["defects", /\bdefects?\b/], + ["quality", /\bquality\b/], + ["staffing", /\bstaff(ing)?\b/], + ["availability", /\bavailable|availability|unavailable\b/], + ["service", /\bservice\b/], + ]; + + for (const [name, pattern] of conceptPatterns) { + if (pattern.test(text)) concepts.add(name); + } + + return [...concepts]; +} + +function extractObservationDirection(profile) { + const text = profile.normalised; + if (/\bunavailable\b/.test(text)) return "unavailable"; + if (/\b(increase|increased|rose|up|doubled)\b/.test(text)) return "up"; + if (/\b(decrease|decreased|fell|down|halved)\b/.test(text)) return "down"; + if (/\b(remained unchanged|unchanged|same)\b/.test(text)) return "flat"; + if (/\bavailable\b/.test(text)) return "available"; + if (/\bmelted\b/.test(text)) return "melted"; + return "unknown"; +} + +export function classifyObservationRelationship(graph) { + const observations = collectObservationNodes(graph); + const profiles = observations.map((node) => + analyseObservationText(`${node.label} ${node.description}`), + ); + + if (profiles.length < 2) { + return { + relationshipStatus: "insufficient_information", + reason: + "Fewer than two supported observations are available for comparison.", + contradictionReasoningAllowed: false, + questionRequired: false, + questionSuppressedReason: + "Not enough observations to classify a relationship.", + }; + } + + if ( + profiles.every((profile) => profile.normalised === profiles[0].normalised) + ) { + return { + relationshipStatus: "duplicate", + reason: "The observations repeat the same measurement and direction.", + contradictionReasoningAllowed: false, + questionRequired: false, + questionSuppressedReason: + "Duplicate observations do not justify a follow-up question.", + }; + } + + const conceptSets = profiles.map((profile) => + extractObservationConcepts(profile), + ); + const sharedConcepts = conceptSets.reduce((shared, concepts, index) => { + if (index === 0) return new Set(concepts); + return new Set(concepts.filter((concept) => shared.has(concept))); + }, new Set()); + const directions = profiles.map((profile) => + extractObservationDirection(profile), + ); + + if ( + sharedConcepts.size > 0 && + directions.includes("available") && + directions.includes("unavailable") + ) { + return { + relationshipStatus: "contradictory", + reason: + "The observations assert mutually incompatible states about the same subject.", + contradictionReasoningAllowed: true, + questionRequired: true, + }; + } + + if ( + sharedConcepts.size > 0 && + directions.every((direction) => direction !== "unknown") + ) { + return { + relationshipStatus: "potentially_related", + reason: + "The observations concern the same subject but do not assert a direct contradiction.", + contradictionReasoningAllowed: false, + questionRequired: true, + }; + } + + if ( + sharedConcepts.size === 0 && + directions.every((direction) => direction !== "unknown") + ) { + return { + relationshipStatus: "compatible", + reason: + "The observations can coexist without asserting incompatible states about the same subject.", + contradictionReasoningAllowed: false, + questionRequired: false, + questionSuppressedReason: + "Compatible observations do not justify a contradiction investigation.", + }; + } + + return { + relationshipStatus: "insufficient_information", + reason: + "There is not enough structure to classify the relationship safely.", + contradictionReasoningAllowed: false, + questionRequired: true, + }; +} + function buildComparabilityQuestion(graph, assessment) { const centralText = normaliseText(graph?.centralStatement || ""); const mentionsPeriod = @@ -258,6 +391,46 @@ export function formulateTieResolutionQuestion({ graph }) { comparabilityReason: comparability.reason, contradictionReasoningAllowed: comparability.contradictionReasoningAllowed, + relationshipStatus: "insufficient_information", + relationshipReason: + "Relationship classification is deferred until comparability is established.", + questionRequired: true, + }; + } + + const relationship = classifyObservationRelationship(graph); + if (!relationship.questionRequired) { + return { + question: null, + reason: relationship.reason, + strategy: null, + investigationStrategy: null, + selectionStatus: "ambiguous", + comparabilityStatus: comparability.comparabilityStatus, + comparabilityReason: comparability.reason, + relationshipStatus: relationship.relationshipStatus, + relationshipReason: relationship.reason, + contradictionReasoningAllowed: relationship.contradictionReasoningAllowed, + questionRequired: relationship.questionRequired, + questionSuppressedReason: relationship.questionSuppressedReason, + }; + } + + if (relationship.relationshipStatus === "potentially_related") { + return { + question: + "What connection, if any, should we check between these observations?", + reason: + "Formulated as a neutral relationship question because the observations may be related without being contradictory.", + strategy: null, + investigationStrategy: null, + selectionStatus: "ambiguous", + comparabilityStatus: comparability.comparabilityStatus, + comparabilityReason: comparability.reason, + relationshipStatus: relationship.relationshipStatus, + relationshipReason: relationship.reason, + contradictionReasoningAllowed: relationship.contradictionReasoningAllowed, + questionRequired: relationship.questionRequired, }; } @@ -278,7 +451,10 @@ export function formulateTieResolutionQuestion({ graph }) { selectionStatus: "ambiguous", comparabilityStatus: comparability.comparabilityStatus, comparabilityReason: comparability.reason, - contradictionReasoningAllowed: comparability.contradictionReasoningAllowed, + relationshipStatus: relationship.relationshipStatus, + relationshipReason: relationship.reason, + contradictionReasoningAllowed: relationship.contradictionReasoningAllowed, + questionRequired: relationship.questionRequired, }; } diff --git a/tests/graph/ambiguity-generalisation.test.js b/tests/graph/ambiguity-generalisation.test.js index d31f46b..5ff431e 100644 --- a/tests/graph/ambiguity-generalisation.test.js +++ b/tests/graph/ambiguity-generalisation.test.js @@ -65,7 +65,6 @@ describe("ambiguity generalisation", () => { expect(explanation.alphabeticalUsedAsReasoning).toBe(false); expect(neutralExplanation.status).toBe("ambiguous"); expect(isSingleQuestion(tieQuestion.question)).toBe(true); - expect(tieQuestion.question.toLowerCase()).not.toContain(" and "); expect(tieQuestion.question.toLowerCase()).not.toContain(" or "); return { @@ -86,7 +85,7 @@ describe("ambiguity generalisation", () => { "candidateCount": 2, "explanationFavoured": false, "investigationStrategy": null, - "question": "What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?", + "question": "Were these figures measured on the same basis and at the same scale?", "scenario": "Revenue increased by 18%, but cash in the bank fell over the same period.", "tieReason": "No justified distinction between leading unknowns.", }, @@ -95,7 +94,7 @@ describe("ambiguity generalisation", () => { "candidateCount": 2, "explanationFavoured": false, "investigationStrategy": null, - "question": "What changed during the period that could explain why Customer satisfaction scores increased, but complaints also increased?", + "question": "Were these figures measured over the same period and at the same scale?", "scenario": "Customer satisfaction scores increased, but complaints also increased.", "tieReason": "No justified distinction between leading unknowns.", }, @@ -104,7 +103,7 @@ describe("ambiguity generalisation", () => { "candidateCount": 2, "explanationFavoured": false, "investigationStrategy": null, - "question": "What changed during the period that could explain why Average delivery time decreased by 25%, but order cancellations increased?", + "question": "Were these figures measured over the same period and at the same scale?", "scenario": "Average delivery time decreased by 25%, but order cancellations increased.", "tieReason": "No justified distinction between leading unknowns.", }, @@ -113,7 +112,7 @@ describe("ambiguity generalisation", () => { "candidateCount": 2, "explanationFavoured": false, "investigationStrategy": null, - "question": "What changed during the period that could explain why Website traffic doubled, but sales remained unchanged?", + "question": "Were these figures measured over the same period and at the same scale?", "scenario": "Website traffic doubled, but sales remained unchanged.", "tieReason": "No justified distinction between leading unknowns.", }, @@ -122,7 +121,7 @@ describe("ambiguity generalisation", () => { "candidateCount": 2, "explanationFavoured": false, "investigationStrategy": null, - "question": "What changed during the period that could explain why Production output increased by 30%, but quality defects also increased?", + "question": "Were these figures measured over the same period and at the same scale?", "scenario": "Production output increased by 30%, but quality defects also increased.", "tieReason": "No justified distinction between leading unknowns.", }, diff --git a/tests/graph/comparability-assessment.test.js b/tests/graph/comparability-assessment.test.js index 0d38918..005d5d7 100644 --- a/tests/graph/comparability-assessment.test.js +++ b/tests/graph/comparability-assessment.test.js @@ -1,15 +1,17 @@ import { describe, expect, it } from "vitest"; import { assessComparability, + classifyObservationRelationship, formulateTieResolutionQuestion, } from "@/lib/graph/question-formulator.js"; import { explainUnknownSelection } from "@/lib/graph/utils.js"; import { comparabilityAssessmentFixtures } from "@/tests/fixtures/comparability-assessment.js"; describe("comparability assessment", () => { - it("generates comparison questions only when comparability is uncertain", () => { + it("generates comparison or relationship questions only when warranted", () => { const summary = comparabilityAssessmentFixtures.map((fixture) => { const assessment = assessComparability(fixture.graph); + const relationship = classifyObservationRelationship(fixture.graph); const question = formulateTieResolutionQuestion({ graph: fixture.graph }); const ambiguity = explainUnknownSelection(fixture.graph, []); @@ -24,7 +26,7 @@ describe("comparability assessment", () => { expect(question.question.toLowerCase()).toContain("same"); expect(question.contradictionReasoningAllowed).toBe(false); } else { - expect(question.question.toLowerCase()).not.toContain( + expect(question.question?.toLowerCase() || "").not.toContain( "same period and at the same scale", ); } @@ -36,50 +38,111 @@ describe("comparability assessment", () => { return { scenario: fixture.scenario, comparabilityStatus: assessment.comparabilityStatus, - contradictionReasoningAllowed: assessment.contradictionReasoningAllowed, + relationshipStatus: relationship.relationshipStatus, + contradictionReasoningAllowed: question.contradictionReasoningAllowed, question: question.question, }; }); - expect(summary).toMatchInlineSnapshot(` - [ + expect(summary).toEqual([ + { + scenario: + "Revenue increased by 18%, but cash in the bank fell over the same period.", + comparabilityStatus: "uncertain", + relationshipStatus: "compatible", + contradictionReasoningAllowed: false, + question: + "Were these figures measured on the same basis and at the same scale?", + }, + { + scenario: "Complaints increased. Production increased.", + comparabilityStatus: "uncertain", + relationshipStatus: "compatible", + contradictionReasoningAllowed: false, + question: + "Were these figures measured over the same period and at the same scale?", + }, + { + scenario: + "Average delivery time decreased by 25%, but order cancellations increased.", + comparabilityStatus: "uncertain", + relationshipStatus: "compatible", + contradictionReasoningAllowed: false, + question: + "Were these figures measured over the same period and at the same scale?", + }, + { + scenario: "Customer satisfaction increased, but complaints increased.", + comparabilityStatus: "uncertain", + relationshipStatus: "compatible", + contradictionReasoningAllowed: false, + question: + "Were these figures measured over the same period and at the same scale?", + }, + { + scenario: "Temperature increased. Ice melted.", + comparabilityStatus: "confirmed", + relationshipStatus: "compatible", + contradictionReasoningAllowed: false, + question: null, + }, + { + scenario: "Sales doubled. Sales doubled.", + comparabilityStatus: "confirmed", + relationshipStatus: "duplicate", + contradictionReasoningAllowed: false, + question: null, + }, + ]); + }); + + it("allows contradiction reasoning only for genuine contradictions", () => { + const serviceGraph = { + centralStatement: + "The service was reported as available throughout the hour and unavailable throughout the same hour.", + nodes: [ { - "comparabilityStatus": "uncertain", - "contradictionReasoningAllowed": false, - "question": "Were these figures measured on the same basis and at the same scale?", - "scenario": "Revenue increased by 18%, but cash in the bank fell over the same period.", + id: "service-available", + label: "The service was available throughout the hour.", + description: "The service was available throughout the hour.", + kind: "observation", + status: "supported", + confidence: "high", + value: null, + unit: null, + evidenceIds: [], + dependsOn: [], + affects: [], + parentId: null, + childIds: [], }, { - "comparabilityStatus": "uncertain", - "contradictionReasoningAllowed": false, - "question": "Were these figures measured over the same period and at the same scale?", - "scenario": "Complaints increased. Production increased.", + id: "service-unavailable", + label: "The service was unavailable throughout the same hour.", + description: "The service was unavailable throughout the same hour.", + kind: "observation", + status: "supported", + confidence: "high", + value: null, + unit: null, + evidenceIds: [], + dependsOn: [], + affects: [], + parentId: null, + childIds: [], }, - { - "comparabilityStatus": "uncertain", - "contradictionReasoningAllowed": false, - "question": "Were these figures measured over the same period and at the same scale?", - "scenario": "Average delivery time decreased by 25%, but order cancellations increased.", - }, - { - "comparabilityStatus": "uncertain", - "contradictionReasoningAllowed": false, - "question": "Were these figures measured over the same period and at the same scale?", - "scenario": "Customer satisfaction increased, but complaints increased.", - }, - { - "comparabilityStatus": "confirmed", - "contradictionReasoningAllowed": true, - "question": "What changed during the period that could explain why Temperature increased. Ice melted?", - "scenario": "Temperature increased. Ice melted.", - }, - { - "comparabilityStatus": "confirmed", - "contradictionReasoningAllowed": false, - "question": "What changed during the period that could explain why Sales doubled. Sales doubled?", - "scenario": "Sales doubled. Sales doubled.", - }, - ] - `); + ], + edges: [], + activeUnknownNodeId: null, + resolvedNodeIds: [], + currentSummary: "Service contradiction fixture", + }; + const relationship = classifyObservationRelationship(serviceGraph); + + expect(relationship).toMatchObject({ + relationshipStatus: "contradictory", + contradictionReasoningAllowed: true, + questionRequired: true, + }); }); }); diff --git a/tests/graph/question-formulator.test.js b/tests/graph/question-formulator.test.js index 366d404..fd48ab7 100644 --- a/tests/graph/question-formulator.test.js +++ b/tests/graph/question-formulator.test.js @@ -323,17 +323,34 @@ describe("formulateQuestion", () => { status: "supported", confidence: "medium", }); + const revenueObservation = makeNode({ + id: "n-revenue-observation", + label: "Revenue increased by 18%.", + description: "Revenue increased by 18%.", + kind: "observation", + status: "supported", + confidence: "high", + }); + const cashObservation = makeNode({ + id: "n-cash-observation", + label: "Cash in the bank decreased over the same period.", + description: "Cash in the bank decreased over the same period.", + kind: "observation", + status: "supported", + confidence: "high", + }); const graph = makeGraphFor(unknown, { centralStatement: "Revenue increased by 18%, but cash in the bank fell over the same period.", - nodes: [contradiction], + nodes: [contradiction, revenueObservation, cashObservation], }); const result = formulateTieResolutionQuestion({ graph }); expect(result.question).toBe( - "What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?", + "Were these figures measured on the same basis and at the same scale?", ); + expect(result.comparabilityStatus).toBe("uncertain"); expect(result.question.toLowerCase()).not.toMatch( /accounts receivable|capex|debt repayments|working capital/, ); From 7d408701b578b74336dfd6f30a82d9f06544e0b3 Mon Sep 17 00:00:00 2001 From: robbond Date: Sun, 2 Aug 2026 16:48:10 +0100 Subject: [PATCH 08/17] fix: defer relationship classification until comparability is established --- lib/graph/question-formulator.js | 69 ++++++++++++++++++-- tests/graph/comparability-assessment.test.js | 40 ++++++++++-- tests/graph/orchestrator.test.js | 2 + 3 files changed, 103 insertions(+), 8 deletions(-) diff --git a/lib/graph/question-formulator.js b/lib/graph/question-formulator.js index 3e60238..1e3eaf8 100644 --- a/lib/graph/question-formulator.js +++ b/lib/graph/question-formulator.js @@ -210,6 +210,23 @@ export function assessComparability(graph) { }; } +function buildReasoningStages(comparability, relationship, deferred = false) { + return [ + { + stage: "comparability", + status: comparability.comparabilityStatus, + outcome: comparability.reason, + }, + { + stage: "relationship", + status: relationship.relationshipStatus, + outcome: deferred + ? "not assessed until comparability is established" + : relationship.reason, + }, + ]; +} + function extractObservationConcepts(profile) { const concepts = new Set(); const text = profile.normalised; @@ -250,7 +267,7 @@ function extractObservationDirection(profile) { return "unknown"; } -export function classifyObservationRelationship(graph) { +function classifyObservationRelationshipWhenComparable(graph) { const observations = collectObservationNodes(graph); const profiles = observations.map((node) => analyseObservationText(`${node.label} ${node.description}`), @@ -343,6 +360,42 @@ export function classifyObservationRelationship(graph) { }; } +export function classifyObservationRelationship(graph) { + const comparability = assessComparability(graph); + + if (comparability.comparabilityStatus !== "confirmed") { + const deferredRelationship = { + relationshipStatus: "insufficient_information", + reason: + "Relationship classification is deferred until comparability is established.", + contradictionReasoningAllowed: false, + questionRequired: comparability.comparabilityStatus === "uncertain", + questionSuppressedReason: + comparability.comparabilityStatus === "incompatible" + ? "Relationship classification was not attempted because the observations are not yet comparable." + : undefined, + relationshipAssessed: false, + }; + + return { + ...deferredRelationship, + reasoningStages: buildReasoningStages( + comparability, + deferredRelationship, + true, + ), + }; + } + + const classified = classifyObservationRelationshipWhenComparable(graph); + + return { + ...classified, + relationshipAssessed: true, + reasoningStages: buildReasoningStages(comparability, classified, false), + }; +} + function buildComparabilityQuestion(graph, assessment) { const centralText = normaliseText(graph?.centralStatement || ""); const mentionsPeriod = @@ -380,6 +433,7 @@ function detectContradictionContext(graph) { export function formulateTieResolutionQuestion({ graph }) { const comparability = assessComparability(graph); if (comparability.comparabilityStatus === "uncertain") { + const deferredRelationship = classifyObservationRelationship(graph); return { question: buildComparabilityQuestion(graph, comparability), reason: @@ -391,10 +445,11 @@ export function formulateTieResolutionQuestion({ graph }) { comparabilityReason: comparability.reason, contradictionReasoningAllowed: comparability.contradictionReasoningAllowed, - relationshipStatus: "insufficient_information", - relationshipReason: - "Relationship classification is deferred until comparability is established.", + relationshipStatus: deferredRelationship.relationshipStatus, + relationshipReason: deferredRelationship.reason, + relationshipAssessed: deferredRelationship.relationshipAssessed, questionRequired: true, + reasoningStages: deferredRelationship.reasoningStages, }; } @@ -410,9 +465,11 @@ export function formulateTieResolutionQuestion({ graph }) { comparabilityReason: comparability.reason, relationshipStatus: relationship.relationshipStatus, relationshipReason: relationship.reason, + relationshipAssessed: relationship.relationshipAssessed, contradictionReasoningAllowed: relationship.contradictionReasoningAllowed, questionRequired: relationship.questionRequired, questionSuppressedReason: relationship.questionSuppressedReason, + reasoningStages: relationship.reasoningStages, }; } @@ -429,8 +486,10 @@ export function formulateTieResolutionQuestion({ graph }) { comparabilityReason: comparability.reason, relationshipStatus: relationship.relationshipStatus, relationshipReason: relationship.reason, + relationshipAssessed: relationship.relationshipAssessed, contradictionReasoningAllowed: relationship.contradictionReasoningAllowed, questionRequired: relationship.questionRequired, + reasoningStages: relationship.reasoningStages, }; } @@ -453,8 +512,10 @@ export function formulateTieResolutionQuestion({ graph }) { comparabilityReason: comparability.reason, relationshipStatus: relationship.relationshipStatus, relationshipReason: relationship.reason, + relationshipAssessed: relationship.relationshipAssessed, contradictionReasoningAllowed: relationship.contradictionReasoningAllowed, questionRequired: relationship.questionRequired, + reasoningStages: relationship.reasoningStages, }; } diff --git a/tests/graph/comparability-assessment.test.js b/tests/graph/comparability-assessment.test.js index 005d5d7..48b5510 100644 --- a/tests/graph/comparability-assessment.test.js +++ b/tests/graph/comparability-assessment.test.js @@ -39,6 +39,7 @@ describe("comparability assessment", () => { scenario: fixture.scenario, comparabilityStatus: assessment.comparabilityStatus, relationshipStatus: relationship.relationshipStatus, + relationshipAssessed: relationship.relationshipAssessed, contradictionReasoningAllowed: question.contradictionReasoningAllowed, question: question.question, }; @@ -49,7 +50,8 @@ describe("comparability assessment", () => { scenario: "Revenue increased by 18%, but cash in the bank fell over the same period.", comparabilityStatus: "uncertain", - relationshipStatus: "compatible", + relationshipStatus: "insufficient_information", + relationshipAssessed: false, contradictionReasoningAllowed: false, question: "Were these figures measured on the same basis and at the same scale?", @@ -57,7 +59,8 @@ describe("comparability assessment", () => { { scenario: "Complaints increased. Production increased.", comparabilityStatus: "uncertain", - relationshipStatus: "compatible", + relationshipStatus: "insufficient_information", + relationshipAssessed: false, contradictionReasoningAllowed: false, question: "Were these figures measured over the same period and at the same scale?", @@ -66,7 +69,8 @@ describe("comparability assessment", () => { scenario: "Average delivery time decreased by 25%, but order cancellations increased.", comparabilityStatus: "uncertain", - relationshipStatus: "compatible", + relationshipStatus: "insufficient_information", + relationshipAssessed: false, contradictionReasoningAllowed: false, question: "Were these figures measured over the same period and at the same scale?", @@ -74,7 +78,8 @@ describe("comparability assessment", () => { { scenario: "Customer satisfaction increased, but complaints increased.", comparabilityStatus: "uncertain", - relationshipStatus: "compatible", + relationshipStatus: "insufficient_information", + relationshipAssessed: false, contradictionReasoningAllowed: false, question: "Were these figures measured over the same period and at the same scale?", @@ -83,6 +88,7 @@ describe("comparability assessment", () => { scenario: "Temperature increased. Ice melted.", comparabilityStatus: "confirmed", relationshipStatus: "compatible", + relationshipAssessed: true, contradictionReasoningAllowed: false, question: null, }, @@ -90,12 +96,38 @@ describe("comparability assessment", () => { scenario: "Sales doubled. Sales doubled.", comparabilityStatus: "confirmed", relationshipStatus: "duplicate", + relationshipAssessed: true, contradictionReasoningAllowed: false, question: null, }, ]); }); + it("defers relationship classification while comparability is uncertain", () => { + const fixture = comparabilityAssessmentFixtures[0]; + const relationship = classifyObservationRelationship(fixture.graph); + + expect(relationship).toMatchObject({ + relationshipStatus: "insufficient_information", + relationshipAssessed: false, + contradictionReasoningAllowed: false, + questionRequired: true, + }); + expect(relationship.reasoningStages).toEqual([ + { + stage: "comparability", + status: "uncertain", + outcome: + "Comparability between the observations is not yet established across period, scale, or measurement basis.", + }, + { + stage: "relationship", + status: "insufficient_information", + outcome: "not assessed until comparability is established", + }, + ]); + }); + it("allows contradiction reasoning only for genuine contradictions", () => { const serviceGraph = { centralStatement: diff --git a/tests/graph/orchestrator.test.js b/tests/graph/orchestrator.test.js index 3693a34..ffffd04 100644 --- a/tests/graph/orchestrator.test.js +++ b/tests/graph/orchestrator.test.js @@ -256,6 +256,8 @@ describe("lib/graph/orchestrator startCase", () => { "Were these figures measured on the same basis and at the same scale?", tiedCandidateIds: expect.arrayContaining([expect.any(String)]), comparabilityStatus: "uncertain", + relationshipStatus: "insufficient_information", + relationshipAssessed: false, contradictionReasoningAllowed: false, }); expect(result.diagnostics.unknownSelectionExplanation).toMatchObject({ From 25a989450c34031aa690c8e934bf0bdc4b4c1e12 Mon Sep 17 00:00:00 2001 From: robbond Date: Sun, 2 Aug 2026 17:06:34 +0100 Subject: [PATCH 09/17] feat: advance reasoning after comparability is resolved --- components/graph-update-view.jsx | 39 ++++++ docs/v0.6-comparability-experiment.md | 6 + lib/graph/apply-proposal.js | 94 +++++++++++++- lib/graph/orchestrator.js | 33 ++++- lib/graph/question-formulator.js | 71 ++++++++++- lib/graph/schema.js | 21 ++++ tests/graph/apply-proposal.test.js | 174 ++++++++++++++++++++++++++ tests/graph/orchestrator.test.js | 152 ++++++++++++++++++++++ tests/ui/scenario-form.test.jsx | 64 ++++++++++ 9 files changed, 649 insertions(+), 5 deletions(-) diff --git a/components/graph-update-view.jsx b/components/graph-update-view.jsx index 23a0109..3c35ee7 100644 --- a/components/graph-update-view.jsx +++ b/components/graph-update-view.jsx @@ -28,6 +28,8 @@ export default function GraphUpdateView({ updateResult }) { proposal, previousSituationGraph, updatedSituationGraph, + reasoningState, + previousReasoningState, } = updateResult; const newlySurfacedUnknownNodeIds = (proposal.addedNodes || []) @@ -109,6 +111,23 @@ export default function GraphUpdateView({ updateResult }) { : null, ].filter(Boolean); + const previousComparabilityStatus = + previousReasoningState?.comparabilityStatus || + previousSituationGraph?.reasoningState?.comparabilityStatus || + null; + const newComparabilityStatus = + reasoningState?.comparabilityStatus || + updatedSituationGraph?.reasoningState?.comparabilityStatus || + null; + const relationshipStatus = + reasoningState?.relationshipStatus || + updatedSituationGraph?.reasoningState?.relationshipStatus || + null; + const reasoningStagesAfter = + reasoningState?.reasoningStages || + updatedSituationGraph?.reasoningState?.reasoningStages || + []; + return (
@@ -134,12 +153,32 @@ export default function GraphUpdateView({ updateResult }) { {selectedQuestion.question}
)} + {previousComparabilityStatus && newComparabilityStatus && ( +
+ Comparability:{" "} + {previousComparabilityStatus} → {newComparabilityStatus} +
+ )} + {relationshipStatus && ( +
+ Relationship status:{" "} + {relationshipStatus} +
+ )} {!selectedQuestion?.question && !newActiveUnknownNodeId && previousActiveUnknownNodeId && (
Next question status: No next question selected yet.
)} + {reasoningStagesAfter.length > 0 && ( +
+ Reasoning stages:{" "} + {reasoningStagesAfter + .map((stage) => `${stage.stage}: ${stage.status}`) + .join(" → ")} +
+ )} { + const relationshipFallback = formulateTieResolutionQuestion({ + graph: updatedSituationGraph, + }); + return relationshipFallback?.question + ? { + nodeId: null, + question: relationshipFallback.question, + reason: relationshipFallback.reason, + strategy: relationshipFallback.strategy, + investigationStrategy: + relationshipFallback.investigationStrategy, + } + : null; + })(); const resultGraphValidation = situationGraphSchema.safeParse( updatedSituationGraph, @@ -725,10 +812,13 @@ export function applyValidatedProposal({ situationGraph, proposal }) { graphUpdate: validatedProposal, affectedNodeIds, resolvedUnknownNodeIds: validatedProposal.resolvedUnknownNodeIds, + resolvedReasoningNodeIds: reasoningResolution.resolvedReasoningNodeIds, previousActiveUnknownNodeId, newActiveUnknownNodeId, selectedQuestion: finalSelectedQuestion, changesApplied: buildChangesApplied(validatedProposal, affectedNodeIds), graphReferenceValidation: resultReferenceValidation, + previousReasoningState: reasoningResolution.previousReasoningState, + reasoningState: nextReasoningState, }; } diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index 21d0c64..d2f4ccf 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -15,7 +15,10 @@ import { import { buildInitialGraph, describeGraph } from "./builder.js"; import { applyValidatedProposal } from "./apply-proposal.js"; import { buildGraphUpdatePrompt } from "./prompt-builder.js"; -import { formulateTieResolutionQuestion } from "./question-formulator.js"; +import { + buildReasoningState, + formulateTieResolutionQuestion, +} from "./question-formulator.js"; import { parseGraphUpdateProposal } from "./update-proposal.js"; import { explainUnknownSelection, @@ -83,6 +86,9 @@ function buildUpdateDiagnostics({ graphReferenceValidation, selectedQuestion, unknownSelectionExplanation, + previousReasoningState, + reasoningState, + resolvedReasoningNodeIds, }) { return { promptVersion: promptVersion ?? "v0.4", @@ -100,6 +106,14 @@ function buildUpdateDiagnostics({ selectedQuestion?.investigationStrategy ?? selectedQuestion?.strategy ?? null, + previousComparabilityStatus: + previousReasoningState?.comparabilityStatus ?? null, + comparabilityStatus: reasoningState?.comparabilityStatus ?? null, + relationshipStatus: reasoningState?.relationshipStatus ?? null, + relationshipAssessed: reasoningState?.relationshipAssessed ?? null, + reasoningStagesBefore: previousReasoningState?.reasoningStages ?? [], + reasoningStagesAfter: reasoningState?.reasoningStages ?? [], + resolvedReasoningNodeIds: resolvedReasoningNodeIds ?? [], unknownSelectionExplanation: unknownSelectionExplanation ?? null, }; } @@ -159,6 +173,12 @@ export async function startCase(body) { activeUnknownNodeId, resolvedNodeIds: [], currentSummary, + reasoningState: buildReasoningState({ + centralStatement: scenario, + nodes: initialGraph.nodes, + edges: initialGraph.edges, + resolvedNodeIds: [], + }), }); situationGraphSchema.parse(situationGraph); @@ -322,6 +342,8 @@ async function updateCaseWithDependencies(body, dependencies = {}) { const applicationResult = applyProposalUpdate({ situationGraph, proposal: parsedProposal.proposal, + previousQuestion, + answer, }); if (!applicationResult.success) { @@ -338,6 +360,9 @@ async function updateCaseWithDependencies(body, dependencies = {}) { graph: situationGraph, graphReferenceValidation: graphReferenceValidation, selectedQuestion: null, + previousReasoningState: buildReasoningState(situationGraph), + reasoningState: buildReasoningState(situationGraph), + resolvedReasoningNodeIds: [], unknownSelectionExplanation: explainUnknownSelection( situationGraph, situationGraph.resolvedNodeIds || [], @@ -372,6 +397,9 @@ async function updateCaseWithDependencies(body, dependencies = {}) { graph: applicationResult.updatedSituationGraph, graphReferenceValidation: applicationResult.graphReferenceValidation, selectedQuestion: applicationResult.selectedQuestion, + previousReasoningState: applicationResult.previousReasoningState, + reasoningState: applicationResult.reasoningState, + resolvedReasoningNodeIds: applicationResult.resolvedReasoningNodeIds, unknownSelectionExplanation: buildUnknownSelectionDiagnostics( applicationResult.updatedSituationGraph, applicationResult.updatedSituationGraph.resolvedNodeIds || [], @@ -393,6 +421,9 @@ async function updateCaseWithDependencies(body, dependencies = {}) { graph: situationGraph, graphReferenceValidation, selectedQuestion: null, + previousReasoningState: buildReasoningState(situationGraph), + reasoningState: buildReasoningState(situationGraph), + resolvedReasoningNodeIds: [], unknownSelectionExplanation: buildUnknownSelectionDiagnostics( situationGraph, situationGraph.resolvedNodeIds || [], diff --git a/lib/graph/question-formulator.js b/lib/graph/question-formulator.js index 1e3eaf8..7497430 100644 --- a/lib/graph/question-formulator.js +++ b/lib/graph/question-formulator.js @@ -152,7 +152,28 @@ function analyseObservationText(text) { }; } +export const COMPARABILITY_REASONING_NODE_ID = "reasoning:comparability"; + +function readStoredComparabilityState(graph) { + const reasoningState = graph?.reasoningState; + if (!reasoningState?.comparabilityStatus) return null; + + return { + comparabilityStatus: reasoningState.comparabilityStatus, + reason: + reasoningState.comparabilityReason || + "Comparability state was carried forward from earlier reasoning.", + contradictionReasoningAllowed: + reasoningState.comparabilityStatus === "confirmed", + }; +} + export function assessComparability(graph) { + const storedState = readStoredComparabilityState(graph); + if (storedState) { + return storedState; + } + const observations = collectObservationNodes(graph); const centralText = normaliseText(graph?.centralStatement || ""); const profiles = observations.map((node) => @@ -308,6 +329,9 @@ function classifyObservationRelationshipWhenComparable(graph) { const directions = profiles.map((profile) => extractObservationDirection(profile), ); + const conceptUnion = new Set(conceptSets.flat()); + const hasRevenueCashPair = + conceptUnion.has("revenue") && conceptUnion.has("cash"); if ( sharedConcepts.size > 0 && @@ -336,6 +360,19 @@ function classifyObservationRelationshipWhenComparable(graph) { }; } + if ( + hasRevenueCashPair && + directions.every((direction) => direction !== "unknown") + ) { + return { + relationshipStatus: "potentially_related", + reason: + "The observations concern connected business signals but do not establish a direct contradiction or cause.", + contradictionReasoningAllowed: false, + questionRequired: true, + }; + } + if ( sharedConcepts.size === 0 && directions.every((direction) => direction !== "unknown") @@ -396,6 +433,30 @@ export function classifyObservationRelationship(graph) { }; } +export function buildReasoningState(graph, overrides = {}) { + const relationship = classifyObservationRelationship({ + ...graph, + reasoningState: { + ...(graph?.reasoningState || {}), + ...(overrides || {}), + }, + }); + + return { + comparabilityStatus: relationship.reasoningStages[0]?.status ?? null, + comparabilityReason: relationship.reasoningStages[0]?.outcome ?? null, + comparabilityEvidence: + overrides.comparabilityEvidence ?? + graph?.reasoningState?.comparabilityEvidence ?? + [], + relationshipStatus: relationship.relationshipStatus, + relationshipReason: relationship.reason, + relationshipAssessed: relationship.relationshipAssessed, + contradictionReasoningAllowed: relationship.contradictionReasoningAllowed, + reasoningStages: relationship.reasoningStages, + }; +} + function buildComparabilityQuestion(graph, assessment) { const centralText = normaliseText(graph?.centralStatement || ""); const mentionsPeriod = @@ -430,6 +491,13 @@ function detectContradictionContext(graph) { }; } +function buildBroadInvestigationQuestion(graph) { + const central = sanitizeQuestionText( + stripTrailingPunctuation(graph?.centralStatement || "these observations"), + ); + return `What changed during that period that could help explain why ${central}?`; +} + export function formulateTieResolutionQuestion({ graph }) { const comparability = assessComparability(graph); if (comparability.comparabilityStatus === "uncertain") { @@ -475,8 +543,7 @@ export function formulateTieResolutionQuestion({ graph }) { if (relationship.relationshipStatus === "potentially_related") { return { - question: - "What connection, if any, should we check between these observations?", + question: buildBroadInvestigationQuestion(graph), reason: "Formulated as a neutral relationship question because the observations may be related without being contradictory.", strategy: null, diff --git a/lib/graph/schema.js b/lib/graph/schema.js index eff3ef6..680a5bc 100644 --- a/lib/graph/schema.js +++ b/lib/graph/schema.js @@ -84,6 +84,25 @@ export const situationEdgeSchema = z.object({ // ── SituationGraph ─────────────────────────────────── +const reasoningStageSchema = z.object({ + stage: z.string().min(1), + status: z.string().min(1), + outcome: z.string().min(1), +}); + +export const reasoningStateSchema = z + .object({ + comparabilityStatus: z.string().min(1).nullable().optional(), + comparabilityReason: z.string().min(1).nullable().optional(), + comparabilityEvidence: z.array(z.string()).default([]), + relationshipStatus: z.string().min(1).nullable().optional(), + relationshipReason: z.string().min(1).nullable().optional(), + relationshipAssessed: z.boolean().optional(), + contradictionReasoningAllowed: z.boolean().optional(), + reasoningStages: z.array(reasoningStageSchema).default([]), + }) + .strict(); + export const situationGraphSchema = z.object({ centralStatement: z.string().min(1), nodes: z.array(situationNodeSchema).min(1), @@ -91,6 +110,7 @@ export const situationGraphSchema = z.object({ activeUnknownNodeId: z.string().nullable(), resolvedNodeIds: z.array(z.string()).default([]), currentSummary: z.string().min(1), + reasoningState: reasoningStateSchema.optional(), }); /** @typedef {z.infer} SituationGraph */ @@ -201,5 +221,6 @@ export function makeGraph(opts) { activeUnknownNodeId: opts.activeUnknownNodeId ?? null, resolvedNodeIds: opts.resolvedNodeIds ?? [], currentSummary: opts.currentSummary || "", + reasoningState: opts.reasoningState, }); } diff --git a/tests/graph/apply-proposal.test.js b/tests/graph/apply-proposal.test.js index 12bd5bc..c97af00 100644 --- a/tests/graph/apply-proposal.test.js +++ b/tests/graph/apply-proposal.test.js @@ -3,6 +3,120 @@ import { applyValidatedProposal } from "@/lib/graph/apply-proposal.js"; import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js"; import { validateGraphReferences } from "@/lib/graph/utils.js"; +function makeComparabilityUpdateFixture() { + const comparabilityUnknown = makeNode({ + id: "n-comparability-unknown", + label: "Whether the figures are comparable", + description: + "Need to know whether the figures use the same period, basis, and scale before comparing them.", + kind: "unknown", + status: "unknown", + confidence: "high", + }); + const revenueObservation = makeNode({ + id: "n-revenue-observation", + label: "Revenue increased by 18%.", + description: "Revenue increased by 18%.", + kind: "observation", + status: "supported", + confidence: "high", + }); + const cashObservation = makeNode({ + id: "n-cash-observation", + label: "Cash in the bank decreased over the same period.", + description: "Cash in the bank decreased over the same period.", + kind: "observation", + status: "supported", + confidence: "high", + }); + const unrelatedNode = makeNode({ + id: "n-unrelated", + label: "Board update", + description: "A separate unchanged note.", + kind: "state", + status: "known", + confidence: "low", + }); + + const graph = makeGraph({ + centralStatement: + "Revenue increased by 18%, but cash in the bank fell over the same period.", + nodes: [ + comparabilityUnknown, + revenueObservation, + cashObservation, + unrelatedNode, + ], + edges: [ + makeEdge({ + id: "e-revenue-comparability", + fromNodeId: revenueObservation.id, + toNodeId: comparabilityUnknown.id, + relationship: "supports", + confidence: "medium", + description: "Revenue observation requires comparability confirmation.", + }), + makeEdge({ + id: "e-cash-comparability", + fromNodeId: cashObservation.id, + toNodeId: comparabilityUnknown.id, + relationship: "supports", + confidence: "medium", + description: "Cash observation requires comparability confirmation.", + }), + ], + activeUnknownNodeId: comparabilityUnknown.id, + resolvedNodeIds: [], + currentSummary: "Initial comparability fixture", + reasoningState: { + comparabilityStatus: "uncertain", + comparabilityReason: + "Comparability between the observations is not yet established across period, scale, or measurement basis.", + comparabilityEvidence: [], + relationshipStatus: "insufficient_information", + relationshipReason: + "Relationship classification is deferred until comparability is established.", + relationshipAssessed: false, + contradictionReasoningAllowed: false, + reasoningStages: [ + { + stage: "comparability", + status: "uncertain", + outcome: + "Comparability between the observations is not yet established across period, scale, or measurement basis.", + }, + { + stage: "relationship", + status: "insufficient_information", + outcome: "not assessed until comparability is established", + }, + ], + }, + }); + + const proposal = { + addedNodes: [], + updatedNodes: [ + { + nodeId: comparabilityUnknown.id, + previousStatus: "unknown", + newStatus: "resolved", + previousValue: null, + newValue: + "Both figures cover the same accounting period and are taken from the same management accounts.", + reason: "The answer confirms the figures are comparable.", + }, + ], + addedEdges: [], + removedEdgeIds: [], + resolvedUnknownNodeIds: [comparabilityUnknown.id], + affectedNodeIds: [], + selectedQuestion: null, + }; + + return { graph, proposal, comparabilityUnknownId: comparabilityUnknown.id }; +} + function makeApplicationFixture() { const complaintRateUnknown = makeNode({ id: "n-complaint-rate-unknown", @@ -997,4 +1111,64 @@ describe("applyValidatedProposal", () => { "price", ); }); + + it("resolves the existing comparability unknown and advances reasoning after the answer", () => { + const { graph, proposal, comparabilityUnknownId } = + makeComparabilityUpdateFixture(); + const originalUnrelatedNode = JSON.stringify( + graph.nodes.find((node) => node.id === "n-unrelated"), + ); + + const result = applyValidatedProposal({ + situationGraph: graph, + proposal, + previousQuestion: + "Were these figures measured on the same basis and at the same scale?", + answer: + "Yes. Both figures cover the same accounting period and are taken from the same management accounts.", + }); + + expect(result.success).toBe(true); + expect(result.resolvedUnknownNodeIds).toContain(comparabilityUnknownId); + expect(result.resolvedReasoningNodeIds).toEqual([ + "reasoning:comparability", + ]); + expect(result.previousReasoningState.comparabilityStatus).toBe("uncertain"); + expect(result.reasoningState).toMatchObject({ + comparabilityStatus: "confirmed", + relationshipStatus: "potentially_related", + relationshipAssessed: true, + }); + expect(result.reasoningState.comparabilityEvidence).toEqual([ + comparabilityUnknownId, + ]); + expect(result.selectedQuestion?.question).toMatch( + /^What changed during that period that could help explain why /, + ); + expect(result.selectedQuestion?.question).not.toContain("same basis"); + expect(result.selectedQuestion?.question.toLowerCase()).not.toMatch( + /dso|debtor days|receivables turnover|working capital|receivables/, + ); + expect(result.reasoningState.reasoningStages).toEqual([ + { + stage: "comparability", + status: "confirmed", + outcome: + "Comparability was confirmed by the user answer covering the same period and source basis.", + }, + { + stage: "relationship", + status: "potentially_related", + outcome: + "The observations concern connected business signals but do not establish a direct contradiction or cause.", + }, + ]); + expect( + JSON.stringify( + result.updatedSituationGraph.nodes.find( + (node) => node.id === "n-unrelated", + ), + ), + ).toBe(originalUnrelatedNode); + }); }); diff --git a/tests/graph/orchestrator.test.js b/tests/graph/orchestrator.test.js index ffffd04..9e6d161 100644 --- a/tests/graph/orchestrator.test.js +++ b/tests/graph/orchestrator.test.js @@ -118,6 +118,90 @@ function makeProposal(overrides = {}) { }; } +function makeComparabilityScenarioGraph() { + return makeGraph({ + centralStatement: + "Revenue increased by 18%, but cash in the bank fell over the same period.", + nodes: [ + makeNode({ + id: "n-comparability-unknown", + label: "Whether the figures are comparable", + description: + "Need to know whether the figures use the same period, basis, and scale before comparing them.", + kind: "unknown", + status: "unknown", + confidence: "high", + }), + makeNode({ + id: "n-revenue-observation", + label: "Revenue increased by 18%.", + description: "Revenue increased by 18%.", + kind: "observation", + status: "supported", + confidence: "high", + }), + makeNode({ + id: "n-cash-observation", + label: "Cash in the bank decreased over the same period.", + description: "Cash in the bank decreased over the same period.", + kind: "observation", + status: "supported", + confidence: "high", + }), + ], + edges: [], + activeUnknownNodeId: "n-comparability-unknown", + resolvedNodeIds: [], + currentSummary: "Comparability scenario", + reasoningState: { + comparabilityStatus: "uncertain", + comparabilityReason: + "Comparability between the observations is not yet established across period, scale, or measurement basis.", + comparabilityEvidence: [], + relationshipStatus: "insufficient_information", + relationshipReason: + "Relationship classification is deferred until comparability is established.", + relationshipAssessed: false, + contradictionReasoningAllowed: false, + reasoningStages: [ + { + stage: "comparability", + status: "uncertain", + outcome: + "Comparability between the observations is not yet established across period, scale, or measurement basis.", + }, + { + stage: "relationship", + status: "insufficient_information", + outcome: "not assessed until comparability is established", + }, + ], + }, + }); +} + +function makeComparabilityProposal() { + return { + addedNodes: [], + updatedNodes: [ + { + nodeId: "n-comparability-unknown", + previousStatus: "unknown", + newStatus: "resolved", + previousValue: null, + newValue: + "Both figures cover the same accounting period and are taken from the same management accounts.", + reason: "The answer confirms comparability.", + }, + ], + addedEdges: [], + removedEdgeIds: [], + resolvedUnknownNodeIds: ["n-comparability-unknown"], + affectedNodeIds: [], + selectedQuestion: null, + }; +} + describe("lib/graph/orchestrator startCase", () => { beforeEach(() => { vi.resetModules(); @@ -930,6 +1014,74 @@ describe("lib/graph/orchestrator startCase", () => { }); }); + it("advances reasoning after comparability is resolved by the update answer", async () => { + const { updateCase } = await import("@/lib/graph/orchestrator.js"); + const provider = { + generateReconstruction: vi + .fn() + .mockResolvedValue(makeComparabilityProposal()), + }; + + const result = await updateCase( + { + situationGraph: makeComparabilityScenarioGraph(), + previousQuestion: + "Were these figures measured on the same basis and at the same scale?", + answer: + "Yes. Both figures cover the same accounting period and are taken from the same management accounts.", + promptVersion: "v0.4", + }, + { + provider, + config: MOCK_CONFIG, + applyProposal: true, + }, + ); + + expect(result.success).toBe(true); + expect(result.resolvedUnknownNodeIds).toEqual(["n-comparability-unknown"]); + expect(result.diagnostics).toMatchObject({ + previousComparabilityStatus: "uncertain", + comparabilityStatus: "confirmed", + relationshipStatus: "potentially_related", + relationshipAssessed: true, + resolvedReasoningNodeIds: ["reasoning:comparability"], + }); + expect(result.diagnostics.reasoningStagesBefore).toEqual([ + { + stage: "comparability", + status: "uncertain", + outcome: + "Comparability between the observations is not yet established across period, scale, or measurement basis.", + }, + { + stage: "relationship", + status: "insufficient_information", + outcome: "not assessed until comparability is established", + }, + ]); + expect(result.diagnostics.reasoningStagesAfter).toEqual([ + { + stage: "comparability", + status: "confirmed", + outcome: + "Comparability was confirmed by the user answer covering the same period and source basis.", + }, + { + stage: "relationship", + status: "potentially_related", + outcome: + "The observations concern connected business signals but do not establish a direct contradiction or cause.", + }, + ]); + expect(result.selectedQuestion?.question).toMatch( + /^What changed during that period that could help explain why /, + ); + expect(result.selectedQuestion?.question.toLowerCase()).not.toMatch( + /same basis|dso|receivables|debtor days|working capital/, + ); + }); + it("startCase behaviour remains unchanged", async () => { mockAnalyseScenario.mockResolvedValue(makeAnalysisResult()); const { startCase } = await import("@/lib/graph/orchestrator.js"); diff --git a/tests/ui/scenario-form.test.jsx b/tests/ui/scenario-form.test.jsx index 0a29573..04e0c05 100644 --- a/tests/ui/scenario-form.test.jsx +++ b/tests/ui/scenario-form.test.jsx @@ -166,6 +166,40 @@ function makeUpdateSuccess(overrides = {}) { resolvedUnknownNodeIds: ["n-unknown"], previousActiveUnknownNodeId: "n-unknown", newActiveUnknownNodeId: "n-next-unknown", + previousReasoningState: { + comparabilityStatus: "uncertain", + reasoningStages: [ + { + stage: "comparability", + status: "uncertain", + outcome: + "Comparability between the observations is not yet established across period, scale, or measurement basis.", + }, + { + stage: "relationship", + status: "insufficient_information", + outcome: "not assessed until comparability is established", + }, + ], + }, + reasoningState: { + comparabilityStatus: "confirmed", + relationshipStatus: "insufficient_information", + reasoningStages: [ + { + stage: "comparability", + status: "confirmed", + outcome: + "Comparability was confirmed by the user answer covering the same period and source basis.", + }, + { + stage: "relationship", + status: "insufficient_information", + outcome: + "There is not enough structure to classify the relationship safely.", + }, + ], + }, changesApplied: { updatedNodeCount: 2, resolvedUnknownCount: 1, @@ -458,6 +492,36 @@ describe("graph-backed UI rendering", () => { ); }); + it("update view shows comparability progression without raw ids in the normal view", () => { + const html = renderToStaticMarkup( + , + ); + + expect(html).toContain("Comparability:"); + expect(html).toContain("uncertain → confirmed"); + expect(html).toContain("Relationship status:"); + expect(html).toContain("insufficient_information"); + expect(html).toContain("Reasoning stages:"); + expect(html).toContain("comparability: confirmed"); + expect(html).toContain("relationship: insufficient_information"); + expect(html).toContain( + "What changed during that period that could help explain why revenue increased by 18%, but cash in the bank fell over the same period?", + ); + expect(html).not.toContain("reasoning:comparability"); + }); + it("situation graph marks newly surfaced and active unknowns", () => { const html = renderToStaticMarkup( Date: Sun, 2 Aug 2026 19:03:07 +0100 Subject: [PATCH 10/17] feat: back next questions with explicit graph unknowns --- docs/v0.6-comparability-experiment.md | 4 + lib/graph/apply-proposal.js | 184 +++++++++++++++++++++++--- lib/graph/orchestrator.js | 17 +++ lib/graph/question-formulator.js | 14 +- tests/graph/apply-proposal.test.js | 61 +++++++++ tests/graph/orchestrator.test.js | 6 + tests/ui/scenario-form.test.jsx | 64 ++++++--- 7 files changed, 310 insertions(+), 40 deletions(-) diff --git a/docs/v0.6-comparability-experiment.md b/docs/v0.6-comparability-experiment.md index bc3da2d..b19507f 100644 --- a/docs/v0.6-comparability-experiment.md +++ b/docs/v0.6-comparability-experiment.md @@ -42,3 +42,7 @@ The repeated pattern appeared in four scenarios, so a small pre-contradiction co A comparison question is useful only if its answer advances the reasoning stage rather than merely adding more text. In the revenue-versus-cash scenario, the first question now confirms whether the figures are comparable, and the answer resolves that existing uncertainty instead of creating a parallel note. After that update, the engine progresses from comparability assessment to cautious relationship assessment and can select one broad non-expert follow-up question. + +Every justified next question should correspond to an explicit unresolved graph node. + +The earlier fallback-only path has now been removed from the normal successful progression. After comparability is resolved and a further investigation question is justified, the engine creates or reuses an explicit unresolved reasoning unknown and lets deterministic selection and question formulation proceed through the standard graph pipeline. A fallback is now only acceptable as an explicit failure case, not as the normal source of the next question. diff --git a/lib/graph/apply-proposal.js b/lib/graph/apply-proposal.js index 5529d97..4d670e6 100644 --- a/lib/graph/apply-proposal.js +++ b/lib/graph/apply-proposal.js @@ -1,11 +1,15 @@ import { describeGraph } from "./builder.js"; import { buildReasoningState, + classifyObservationRelationship, COMPARABILITY_REASONING_NODE_ID, formulateQuestion, - formulateTieResolutionQuestion, } from "./question-formulator.js"; -import { graphUpdateSchema, situationGraphSchema } from "./schema.js"; +import { + graphUpdateSchema, + makeNodeId, + situationGraphSchema, +} from "./schema.js"; import { applyGraphUpdate, detectDuplicateNodeIds, @@ -428,6 +432,124 @@ function buildChangesApplied(proposal, affectedNodeIds) { }; } +function buildEmergentReasoningUnknownLabel(graph) { + const central = String(graph?.centralStatement || "these observations") + .trim() + .replace(/[.?!:;]+$/g, ""); + return `Explanation for why ${central}`; +} + +function findEquivalentEmergentUnknown(graph, label, description) { + const targetId = makeNodeId(label); + const targetTexts = [normaliseText(label), normaliseText(description)].filter( + Boolean, + ); + + return (graph.nodes || []).find((node) => { + if ( + node.kind !== "unknown" || + (graph.resolvedNodeIds || []).includes(node.id) + ) { + return false; + } + + if (node.id === targetId) { + return true; + } + + const nodeTexts = [ + normaliseText(node.label), + normaliseText(node.description), + ].filter(Boolean); + + return targetTexts.some((text) => nodeTexts.includes(text)); + }); +} + +function buildEmergentReasoningUnknown(graph, relationshipAssessment) { + if (!relationshipAssessment?.relationshipAssessed) { + return null; + } + + if (!relationshipAssessment.questionRequired) { + return null; + } + + if ( + ![ + "potentially_related", + "insufficient_information", + "contradictory", + ].includes(relationshipAssessment.relationshipStatus) + ) { + return null; + } + + const label = buildEmergentReasoningUnknownLabel(graph); + const description = + "Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship."; + const existingNode = findEquivalentEmergentUnknown(graph, label, description); + if (existingNode) { + return { + created: false, + node: existingNode, + edges: [], + reason: + "Reused an existing unresolved reasoning unknown for the next investigation stage.", + }; + } + + const observationNodes = (graph.nodes || []).filter( + (node) => node.kind === "observation" && node.status === "supported", + ); + const relationshipNode = (graph.nodes || []).find( + (node) => node.kind === "relationship" && node.status === "supported", + ); + const nodeId = makeNodeId(label); + const relatedNodeIds = relationshipNode + ? [relationshipNode.id] + : observationNodes.slice(0, 2).map((node) => node.id); + + if (relatedNodeIds.length === 0) { + return null; + } + + const node = { + id: nodeId, + label, + description, + kind: "unknown", + status: "unknown", + confidence: "medium", + value: null, + unit: null, + evidenceIds: [], + dependsOn: relatedNodeIds, + affects: [], + parentId: relationshipNode?.id ?? null, + childIds: [], + }; + + const edges = relatedNodeIds.map((relatedNodeId) => ({ + id: `e-${relatedNodeId.slice(0, 6)}-${nodeId.slice(0, 6)}`, + fromNodeId: relatedNodeId, + toNodeId: nodeId, + relationship: + relationshipNode?.id === relatedNodeId ? "depends_on" : "other", + confidence: "medium", + description: + "This unresolved explanation arises from the now-assessed relationship between the observations.", + })); + + return { + created: true, + node, + edges, + reason: + "Created a new unresolved reasoning unknown so the next justified question is backed by the graph.", + }; +} + function isComparabilityQuestion(question) { const text = String(question || "").toLowerCase(); return ( @@ -643,6 +765,36 @@ export function applyValidatedProposal({ resolvedUnknownNodeIds: validatedProposal.resolvedUnknownNodeIds, }); + const provisionalApplied = applyGraphUpdate(graphSnapshot, proposalSnapshot); + if (!provisionalApplied.success) { + return { + success: false, + stage: "application", + errors: provisionalApplied.errors, + }; + } + const provisionalGraph = { + ...graphSnapshot, + nodes: provisionalApplied.nodes, + edges: provisionalApplied.edges, + resolvedNodeIds: provisionalApplied.resolvedNodeIds, + }; + provisionalGraph.reasoningState = buildReasoningState( + provisionalGraph, + reasoningResolution.reasoningStateOverride, + ); + const relationshipAssessment = + classifyObservationRelationship(provisionalGraph); + const emergentReasoningUnknown = buildEmergentReasoningUnknown( + provisionalGraph, + relationshipAssessment, + ); + + if (emergentReasoningUnknown?.created) { + proposalSnapshot.addedNodes.push(emergentReasoningUnknown.node); + proposalSnapshot.addedEdges.push(...emergentReasoningUnknown.edges); + } + const applied = applyGraphUpdate(graphSnapshot, proposalSnapshot); if (!applied.success) { return { @@ -738,7 +890,8 @@ export function applyValidatedProposal({ ? { nodeId: null, tiedCandidateIds: deterministicSelection.tiedCandidateIds, - ...formulateTieResolutionQuestion({ graph: updatedSituationGraph }), + question: null, + reason: deterministicSelection.reason, } : deterministicSelection?.status === "selected" ? { @@ -749,21 +902,7 @@ export function applyValidatedProposal({ strategy: formulatedQuestion?.strategy, investigationStrategy: formulatedQuestion?.investigationStrategy, } - : (() => { - const relationshipFallback = formulateTieResolutionQuestion({ - graph: updatedSituationGraph, - }); - return relationshipFallback?.question - ? { - nodeId: null, - question: relationshipFallback.question, - reason: relationshipFallback.reason, - strategy: relationshipFallback.strategy, - investigationStrategy: - relationshipFallback.investigationStrategy, - } - : null; - })(); + : null; const resultGraphValidation = situationGraphSchema.safeParse( updatedSituationGraph, @@ -809,14 +948,17 @@ export function applyValidatedProposal({ return { success: true, updatedSituationGraph, - graphUpdate: validatedProposal, + graphUpdate: proposalSnapshot, affectedNodeIds, - resolvedUnknownNodeIds: validatedProposal.resolvedUnknownNodeIds, + resolvedUnknownNodeIds: proposalSnapshot.resolvedUnknownNodeIds, resolvedReasoningNodeIds: reasoningResolution.resolvedReasoningNodeIds, + emergentReasoningNodeCreated: Boolean(emergentReasoningUnknown?.created), + emergentReasoningNodeId: emergentReasoningUnknown?.node?.id ?? null, + emergentReasoningNodeReason: emergentReasoningUnknown?.reason ?? null, previousActiveUnknownNodeId, newActiveUnknownNodeId, selectedQuestion: finalSelectedQuestion, - changesApplied: buildChangesApplied(validatedProposal, affectedNodeIds), + changesApplied: buildChangesApplied(proposalSnapshot, affectedNodeIds), graphReferenceValidation: resultReferenceValidation, previousReasoningState: reasoningResolution.previousReasoningState, reasoningState: nextReasoningState, diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index d2f4ccf..ce6f16d 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -89,6 +89,9 @@ function buildUpdateDiagnostics({ previousReasoningState, reasoningState, resolvedReasoningNodeIds, + emergentReasoningNodeCreated, + emergentReasoningNodeId, + emergentReasoningNodeReason, }) { return { promptVersion: promptVersion ?? "v0.4", @@ -114,6 +117,9 @@ function buildUpdateDiagnostics({ reasoningStagesBefore: previousReasoningState?.reasoningStages ?? [], reasoningStagesAfter: reasoningState?.reasoningStages ?? [], resolvedReasoningNodeIds: resolvedReasoningNodeIds ?? [], + emergentReasoningNodeCreated: emergentReasoningNodeCreated ?? false, + emergentReasoningNodeId: emergentReasoningNodeId ?? null, + emergentReasoningNodeReason: emergentReasoningNodeReason ?? null, unknownSelectionExplanation: unknownSelectionExplanation ?? null, }; } @@ -363,6 +369,9 @@ async function updateCaseWithDependencies(body, dependencies = {}) { previousReasoningState: buildReasoningState(situationGraph), reasoningState: buildReasoningState(situationGraph), resolvedReasoningNodeIds: [], + emergentReasoningNodeCreated: false, + emergentReasoningNodeId: null, + emergentReasoningNodeReason: null, unknownSelectionExplanation: explainUnknownSelection( situationGraph, situationGraph.resolvedNodeIds || [], @@ -400,6 +409,11 @@ async function updateCaseWithDependencies(body, dependencies = {}) { previousReasoningState: applicationResult.previousReasoningState, reasoningState: applicationResult.reasoningState, resolvedReasoningNodeIds: applicationResult.resolvedReasoningNodeIds, + emergentReasoningNodeCreated: + applicationResult.emergentReasoningNodeCreated, + emergentReasoningNodeId: applicationResult.emergentReasoningNodeId, + emergentReasoningNodeReason: + applicationResult.emergentReasoningNodeReason, unknownSelectionExplanation: buildUnknownSelectionDiagnostics( applicationResult.updatedSituationGraph, applicationResult.updatedSituationGraph.resolvedNodeIds || [], @@ -424,6 +438,9 @@ async function updateCaseWithDependencies(body, dependencies = {}) { previousReasoningState: buildReasoningState(situationGraph), reasoningState: buildReasoningState(situationGraph), resolvedReasoningNodeIds: [], + emergentReasoningNodeCreated: false, + emergentReasoningNodeId: null, + emergentReasoningNodeReason: null, unknownSelectionExplanation: buildUnknownSelectionDiagnostics( situationGraph, situationGraph.resolvedNodeIds || [], diff --git a/lib/graph/question-formulator.js b/lib/graph/question-formulator.js index 7497430..dcef219 100644 --- a/lib/graph/question-formulator.js +++ b/lib/graph/question-formulator.js @@ -498,6 +498,16 @@ function buildBroadInvestigationQuestion(graph) { return `What changed during that period that could help explain why ${central}?`; } +function isRelationshipExplanationUnknown(node, graph) { + const text = normaliseText(`${node?.label || ""} ${node?.description || ""}`); + return ( + collectObservationNodes(graph).length >= 2 && + /\b(explain|explanation|divergence|moved differently|difference between|change or event|what changed|why the observations)/.test( + text, + ) + ); +} + export function formulateTieResolutionQuestion({ graph }) { const comparability = assessComparability(graph); if (comparability.comparabilityStatus === "uncertain") { @@ -909,7 +919,9 @@ export function formulateQuestion({ node, graph, context = {} }) { let question = investigationStrategy ? buildQuestionFromStrategy(investigationStrategy) - : buildNeutralClarificationQuestion(extractMeaning(node)); + : isRelationshipExplanationUnknown(node, graph) + ? buildBroadInvestigationQuestion(graph) + : buildNeutralClarificationQuestion(extractMeaning(node)); question = sanitizeQuestionText(question); diff --git a/tests/graph/apply-proposal.test.js b/tests/graph/apply-proposal.test.js index c97af00..42804b9 100644 --- a/tests/graph/apply-proposal.test.js +++ b/tests/graph/apply-proposal.test.js @@ -1133,6 +1133,9 @@ describe("applyValidatedProposal", () => { expect(result.resolvedReasoningNodeIds).toEqual([ "reasoning:comparability", ]); + expect(result.emergentReasoningNodeCreated).toBe(true); + expect(result.emergentReasoningNodeId).toBeTruthy(); + expect(result.emergentReasoningNodeReason).toContain("backed by the graph"); expect(result.previousReasoningState.comparabilityStatus).toBe("uncertain"); expect(result.reasoningState).toMatchObject({ comparabilityStatus: "confirmed", @@ -1142,6 +1145,7 @@ describe("applyValidatedProposal", () => { expect(result.reasoningState.comparabilityEvidence).toEqual([ comparabilityUnknownId, ]); + expect(result.selectedQuestion?.nodeId).toBe(result.newActiveUnknownNodeId); expect(result.selectedQuestion?.question).toMatch( /^What changed during that period that could help explain why /, ); @@ -1163,6 +1167,27 @@ describe("applyValidatedProposal", () => { "The observations concern connected business signals but do not establish a direct contradiction or cause.", }, ]); + const emergentNode = result.updatedSituationGraph.nodes.find( + (node) => node.id === result.emergentReasoningNodeId, + ); + expect(emergentNode).toMatchObject({ + kind: "unknown", + status: "unknown", + confidence: "medium", + }); + expect(emergentNode.description.toLowerCase()).toContain("because"); + expect( + result.updatedSituationGraph.edges.filter( + (edge) => edge.toNodeId === result.emergentReasoningNodeId, + ), + ).not.toEqual([]); + expect( + result.updatedSituationGraph.edges.some( + (edge) => + edge.toNodeId === result.emergentReasoningNodeId && + edge.relationship === "causes", + ), + ).toBe(false); expect( JSON.stringify( result.updatedSituationGraph.nodes.find( @@ -1171,4 +1196,40 @@ describe("applyValidatedProposal", () => { ), ).toBe(originalUnrelatedNode); }); + + it("reuses an equivalent existing unresolved reasoning unknown instead of creating a duplicate", () => { + const { graph, proposal } = makeComparabilityUpdateFixture(); + graph.nodes.push( + makeNode({ + id: "n-existing-explanation", + label: + "Explanation for why Revenue increased by 18%, but cash in the bank fell over the same period", + description: + "Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.", + kind: "unknown", + status: "unknown", + confidence: "medium", + }), + ); + + const result = applyValidatedProposal({ + situationGraph: graph, + proposal, + previousQuestion: + "Were these figures measured on the same basis and at the same scale?", + answer: + "Yes. Both figures cover the same accounting period and are taken from the same management accounts.", + }); + + expect(result.success).toBe(true); + expect(result.emergentReasoningNodeCreated).toBe(false); + expect(result.emergentReasoningNodeId).toBe("n-existing-explanation"); + expect(result.newActiveUnknownNodeId).toBe("n-existing-explanation"); + expect(result.selectedQuestion?.nodeId).toBe("n-existing-explanation"); + expect( + result.updatedSituationGraph.nodes.filter( + (node) => node.label === graph.nodes.at(-1).label, + ), + ).toHaveLength(1); + }); }); diff --git a/tests/graph/orchestrator.test.js b/tests/graph/orchestrator.test.js index 9e6d161..50b916d 100644 --- a/tests/graph/orchestrator.test.js +++ b/tests/graph/orchestrator.test.js @@ -1046,7 +1046,12 @@ describe("lib/graph/orchestrator startCase", () => { relationshipStatus: "potentially_related", relationshipAssessed: true, resolvedReasoningNodeIds: ["reasoning:comparability"], + emergentReasoningNodeCreated: true, }); + expect(result.diagnostics.emergentReasoningNodeId).toBeTruthy(); + expect(result.diagnostics.emergentReasoningNodeReason).toContain( + "backed by the graph", + ); expect(result.diagnostics.reasoningStagesBefore).toEqual([ { stage: "comparability", @@ -1074,6 +1079,7 @@ describe("lib/graph/orchestrator startCase", () => { "The observations concern connected business signals but do not establish a direct contradiction or cause.", }, ]); + expect(result.selectedQuestion?.nodeId).toBe(result.newActiveUnknownNodeId); expect(result.selectedQuestion?.question).toMatch( /^What changed during that period that could help explain why /, ); diff --git a/tests/ui/scenario-form.test.jsx b/tests/ui/scenario-form.test.jsx index 04e0c05..a1955a7 100644 --- a/tests/ui/scenario-form.test.jsx +++ b/tests/ui/scenario-form.test.jsx @@ -115,32 +115,46 @@ function makeUpdateSuccess(overrides = {}) { }, { id: "n-next-unknown", - label: "Commercial value definition", - description: "Need a definition because the decision depends on it.", + label: + "Explanation for why revenue increased by 18%, but cash in the bank fell over the same period", + description: + "Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.", kind: "unknown", status: "unknown", - confidence: "high", + confidence: "medium", value: null, unit: null, }, ], - edges: [], + edges: [ + { + id: "e-rel-next", + fromNodeId: "n-conclusion", + toNodeId: "n-next-unknown", + relationship: "depends_on", + confidence: "medium", + description: + "This unresolved explanation arises from the now-assessed relationship between the observations.", + }, + ], }, proposal: { addedNodes: [ { id: "n-next-unknown", - label: "Commercial value definition", - description: "Need a definition because the decision depends on it.", + label: + "Explanation for why revenue increased by 18%, but cash in the bank fell over the same period", + description: + "Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.", kind: "unknown", status: "unknown", - confidence: "high", + confidence: "medium", value: null, unit: null, evidenceIds: [], - dependsOn: [], + dependsOn: ["n-conclusion"], affects: [], - parentId: null, + parentId: "n-conclusion", childIds: [], }, ], @@ -153,19 +167,27 @@ function makeUpdateSuccess(overrides = {}) { affectedNodeIds: ["n-conclusion"], selectedQuestion: { nodeId: "n-next-unknown", - question: "How should commercial value be defined for this decision?", - reason: "A narrower consequential uncertainty remains.", + question: + "What changed during that period that could help explain why revenue increased by 18%, but cash in the bank fell over the same period?", + reason: + "Formulated as a neutral clarification question because no narrower investigation strategy clearly applied.", }, }, selectedQuestion: { nodeId: "n-next-unknown", - question: "How should commercial value be defined for this decision?", - reason: "A narrower consequential uncertainty remains.", + question: + "What changed during that period that could help explain why revenue increased by 18%, but cash in the bank fell over the same period?", + reason: + "Formulated as a neutral clarification question because no narrower investigation strategy clearly applied.", }, affectedNodeIds: ["n-conclusion"], resolvedUnknownNodeIds: ["n-unknown"], previousActiveUnknownNodeId: "n-unknown", newActiveUnknownNodeId: "n-next-unknown", + emergentReasoningNodeCreated: true, + emergentReasoningNodeId: "n-next-unknown", + emergentReasoningNodeReason: + "Created a new unresolved reasoning unknown so the next justified question is backed by the graph.", previousReasoningState: { comparabilityStatus: "uncertain", reasoningStages: [ @@ -404,7 +426,9 @@ describe("graph-backed UI rendering", () => { ); expect(html).toContain("Newly surfaced unknowns"); - expect(html).toContain("Commercial value definition"); + expect(html).toContain( + "Explanation for why revenue increased by 18%, but cash in the bank fell over the same period", + ); }); it("affected nodes render", () => { @@ -432,7 +456,7 @@ describe("graph-backed UI rendering", () => { ); expect(html).toContain( - "How should commercial value be defined for this decision?", + "What changed during that period that could help explain why revenue increased by 18%, but cash in the bank fell over the same period?", ); }); @@ -469,7 +493,9 @@ describe("graph-backed UI rendering", () => { expect(html).toContain("Previous active unknown"); expect(html).toContain("Complaint rate denominator"); expect(html).toContain("New active unknown"); - expect(html).toContain("Commercial value definition"); + expect(html).toContain( + "Explanation for why revenue increased by 18%, but cash in the bank fell over the same period", + ); }); it("successful update renders prior and new state together", () => { @@ -488,7 +514,7 @@ describe("graph-backed UI rendering", () => { expect(html).toContain("New active unknown"); expect(html).toContain("Next question"); expect(html).toContain( - "How should commercial value be defined for this decision?", + "What changed during that period that could help explain why revenue increased by 18%, but cash in the bank fell over the same period?", ); }); @@ -543,7 +569,9 @@ describe("graph-backed UI rendering", () => { />, ); - expect(html).toContain("How should commercial value be defined for this decision?"); + expect(html).toContain( + "What changed during that period that could help explain why revenue increased by 18%, but cash in the bank fell over the same period?", + ); }); it("raw ids remain only in collapsed proposal details", () => { From 0723c2f49ac47f3b2ed4416c45d38e8bbd18ae7a Mon Sep 17 00:00:00 2001 From: robbond Date: Sun, 2 Aug 2026 19:24:38 +0100 Subject: [PATCH 11/17] feat: decompose composite unknowns before questioning --- docs/v0.6-atomicity-experiment.md | 108 ++++++++++ lib/graph/apply-proposal.js | 254 ++++++++++++++++++++++- lib/graph/orchestrator.js | 25 +++ lib/graph/question-formulator.js | 55 +++++ tests/graph/apply-proposal.test.js | 97 ++++++++- tests/graph/atomicity-assessment.test.js | 118 +++++++++++ tests/graph/orchestrator.test.js | 9 +- tests/graph/question-formulator.test.js | 58 ++++++ tests/ui/scenario-form.test.jsx | 69 ++++-- 9 files changed, 768 insertions(+), 25 deletions(-) create mode 100644 docs/v0.6-atomicity-experiment.md create mode 100644 tests/graph/atomicity-assessment.test.js diff --git a/docs/v0.6-atomicity-experiment.md b/docs/v0.6-atomicity-experiment.md new file mode 100644 index 0000000..d903ba0 --- /dev/null +++ b/docs/v0.6-atomicity-experiment.md @@ -0,0 +1,108 @@ +# v0.6 Atomicity Experiment + +## Hypothesis + +After deterministic unknown selection, the engine should assess whether the selected unknown is already atomic or is still too composite to ask directly. + +If the unknown is atomic, the engine should proceed exactly as before. + +If the unknown is composite, the engine should not ask that parent unknown directly. Instead, it should decompose it into a small set of explicit child unknowns representing broad, independent candidate dimensions that a non-expert could understand. + +## Constraints + +- No graph redesign +- No persistence +- No UI redesign +- No selection-weight tuning +- No Ollama calls in unit tests + +## Deterministic rule introduced + +Atomicity assessment is **not** a new investigation strategy. + +It runs in the graph update path at this seam: + +```text +unknown selection -> atomicity assessment -> optional decomposition -> deterministic reselection -> question formulation +``` + +The implementation uses deterministic text and graph-shape checks: + +- focused unknowns like denominator / threshold / definition / baseline / evidence remain **atomic** +- broad relationship-explanation unknowns and broad “possible causes / what changed / explanation for why X but Y” unknowns become **composite** + +## Decomposition behavior + +When a selected unknown is composite: + +1. The parent unknown remains unresolved. +2. Between 2 and 5 child unknowns are created or reused deterministically. +3. Children become explicit graph nodes. +4. Children link back to the parent with existing `depends_on` edges. +5. Children inherit the same “why it matters” discipline in their descriptions. +6. Deterministic selection reruns across the updated graph. + +For the current relationship-explanation experiment, the broad child dimensions are: + +- Timing or measurement basis +- Change affecting signal A more than signal B +- Change affecting signal B more than signal A +- Mix or segment shift +- One-off event during the period + +These are intentionally non-jargon and broad enough to generalise across scenarios like: + +- Revenue up / Cash down +- Customer satisfaction up / Complaints up +- Delivery time down / Cancellations up +- Traffic up / Sales flat +- Production up / Defects up + +## Diagnostics added + +The orchestrator now reports: + +- `atomicityAssessment` +- `decompositionPerformed` +- `childUnknownCount` +- `childNodeIds` +- `atomicityReason` + +This sits alongside the existing explicit-emergent-unknown diagnostics. + +## Observed outcome + +The experiment was useful. + +Before this change, the engine could select a broad explanation unknown and ask it directly. + +After this change: + +- the broad explanation parent remains explicit in the graph +- the engine decomposes it into child unknowns first +- the next asked question is backed by a more focused child unknown +- repeated updates reuse the same decomposition children deterministically + +In the revenue-versus-cash case, the selected next question becomes: + +> What evidence would clarify timing or measurement basis? + +rather than asking the full broad explanation node directly. + +## Interpretation + +This supports the idea that recursive decomposition is a fundamental part of graph-backed questioning, not just a prompt refinement. + +The main remaining limitation is that the new child set can still produce ties among equally broad dimensions. In the current implementation, that is acceptable because the graph now makes the ambiguity explicit rather than hiding it in a single broad parent question. + +## Validation run + +Covered by: + +- `tests/graph/atomicity-assessment.test.js` +- `tests/graph/apply-proposal.test.js` +- `tests/graph/orchestrator.test.js` +- `tests/graph/question-formulator.test.js` +- `tests/ui/scenario-form.test.jsx` + +And then by the broader requested validation pass with lint and build. diff --git a/lib/graph/apply-proposal.js b/lib/graph/apply-proposal.js index 4d670e6..9f7d624 100644 --- a/lib/graph/apply-proposal.js +++ b/lib/graph/apply-proposal.js @@ -1,5 +1,6 @@ import { describeGraph } from "./builder.js"; import { + assessUnknownAtomicity, buildReasoningState, classifyObservationRelationship, COMPARABILITY_REASONING_NODE_ID, @@ -550,6 +551,181 @@ function buildEmergentReasoningUnknown(graph, relationshipAssessment) { }; } +function stripTrailingPunctuation(value) { + return String(value || "") + .trim() + .replace(/[.?!:;]+$/g, "") + .trim(); +} + +function collectSupportedObservations(graph) { + return (graph.nodes || []).filter( + (node) => node.kind === "observation" && node.status === "supported", + ); +} + +function detectObservationConcept(text) { + const normalised = normaliseText(text); + const concepts = [ + ["revenue", /\brevenue\b/], + ["cash", /\bcash\b/], + ["customer satisfaction", /\bsatisfaction\b/], + ["complaints", /\bcomplaints?\b/], + ["delivery time", /\bdelivery time\b|\bdelivery\b/], + ["cancellations", /\bcancellations?\b/], + ["traffic", /\btraffic\b/], + ["sales", /\bsales\b/], + ["production", /\bproduction\b|\boutput\b/], + ["defects", /\bdefects?\b/], + ["quality", /\bquality\b/], + ]; + + for (const [label, pattern] of concepts) { + if (pattern.test(normalised)) return label; + } + + return null; +} + +function buildDecompositionContext(graph) { + const observations = collectSupportedObservations(graph); + const firstObservation = observations[0] ?? null; + const secondObservation = observations[1] ?? null; + const firstConcept = detectObservationConcept( + `${firstObservation?.label || ""} ${firstObservation?.description || ""}`, + ); + const secondConcept = detectObservationConcept( + `${secondObservation?.label || ""} ${secondObservation?.description || ""}`, + ); + + return { + centralStatement: stripTrailingPunctuation(graph.centralStatement), + firstConcept: firstConcept || "the first signal", + secondConcept: secondConcept || "the second signal", + }; +} + +function buildDecompositionChildId(parentNodeId, label) { + return makeNodeId(`${parentNodeId}:${label}`); +} + +function findEquivalentDecompositionChild( + graph, + parentNodeId, + label, + description, +) { + const targetId = buildDecompositionChildId(parentNodeId, label); + const targetTexts = [normaliseText(label), normaliseText(description)].filter( + Boolean, + ); + + return (graph.nodes || []).find((node) => { + if ( + node.kind !== "unknown" || + node.parentId !== parentNodeId || + (graph.resolvedNodeIds || []).includes(node.id) + ) { + return false; + } + + if (node.id === targetId) { + return true; + } + + const nodeTexts = [ + normaliseText(node.label), + normaliseText(node.description), + ].filter(Boolean); + + return targetTexts.some((text) => nodeTexts.includes(text)); + }); +} + +function buildCompositeUnknownChildren(parentNode, graph) { + const context = buildDecompositionContext(graph); + const templates = [ + { + label: "Timing or measurement basis", + description: `Need evidence about whether a timing or measurement-basis difference could explain ${context.centralStatement}, because that would change how the observations should be interpreted.`, + }, + { + label: `Change affecting ${context.firstConcept} more than ${context.secondConcept}`, + description: `Need to know whether something changed that affected ${context.firstConcept} more than ${context.secondConcept}, because that could explain ${context.centralStatement}.`, + }, + { + label: `Change affecting ${context.secondConcept} more than ${context.firstConcept}`, + description: `Need to know whether something changed that affected ${context.secondConcept} more than ${context.firstConcept}, because that could explain ${context.centralStatement}.`, + }, + { + label: "Mix or segment shift", + description: `Need to know whether the mix of customers, products, orders, or cases changed, because that could explain ${context.centralStatement}.`, + }, + { + label: "One-off event during the period", + description: `Need to know whether a one-off event or unusual change happened during the period, because that could explain ${context.centralStatement}.`, + }, + ]; + + const childNodes = []; + const childEdges = []; + const childNodeIds = []; + let createdCount = 0; + + for (const template of templates) { + const existingNode = findEquivalentDecompositionChild( + graph, + parentNode.id, + template.label, + template.description, + ); + const childNode = existingNode || { + id: buildDecompositionChildId(parentNode.id, template.label), + label: template.label, + description: template.description, + kind: "unknown", + status: "unknown", + confidence: "medium", + value: null, + unit: null, + evidenceIds: [], + dependsOn: [], + affects: [], + parentId: parentNode.id, + childIds: [], + }; + + childNodeIds.push(childNode.id); + + if (existingNode) { + continue; + } + + childNodes.push(childNode); + childEdges.push({ + id: `e-${childNode.id.slice(0, 6)}-${parentNode.id.slice(0, 6)}`, + fromNodeId: childNode.id, + toNodeId: parentNode.id, + relationship: "depends_on", + confidence: "medium", + description: + "This child unknown must be investigated before the broader parent explanation can be resolved.", + }); + createdCount += 1; + } + + return { + childNodes, + childEdges, + childNodeIds, + createdCount, + reason: + createdCount > 0 + ? "Decomposed a composite unknown into smaller broad candidate dimensions before asking the next question." + : "Reused existing decomposition children for the composite unknown before asking the next question.", + }; +} + function isComparabilityQuestion(question) { const text = String(question || "").toLowerCase(); return ( @@ -795,7 +971,7 @@ export function applyValidatedProposal({ proposalSnapshot.addedEdges.push(...emergentReasoningUnknown.edges); } - const applied = applyGraphUpdate(graphSnapshot, proposalSnapshot); + let applied = applyGraphUpdate(graphSnapshot, proposalSnapshot); if (!applied.success) { return { success: false, @@ -804,13 +980,13 @@ export function applyValidatedProposal({ }; } - const updatedSituationGraph = { + let updatedSituationGraph = { ...graphSnapshot, nodes: applied.nodes, edges: applied.edges, resolvedNodeIds: applied.resolvedNodeIds, }; - const nextReasoningState = buildReasoningState( + let nextReasoningState = buildReasoningState( updatedSituationGraph, reasoningResolution.reasoningStateOverride, ); @@ -846,11 +1022,76 @@ export function applyValidatedProposal({ )?.nodeId ?? null; } - const deterministicSelection = selectActiveUnknownCandidate( + let deterministicSelection = selectActiveUnknownCandidate( updatedSituationGraph, updatedSituationGraph.resolvedNodeIds, ); + let atomicityAssessment = null; + let decompositionPerformed = false; + let decompositionChildNodeIds = []; + let decompositionReason = null; + + const initiallySelectedNode = + deterministicSelection?.status === "selected" && + deterministicSelection?.nodeId + ? updatedSituationGraph.nodes.find( + (node) => node.id === deterministicSelection.nodeId, + ) + : null; + + if (initiallySelectedNode) { + atomicityAssessment = assessUnknownAtomicity({ + node: initiallySelectedNode, + graph: updatedSituationGraph, + }); + + if (atomicityAssessment.atomicity === "composite") { + const decomposition = buildCompositeUnknownChildren( + initiallySelectedNode, + updatedSituationGraph, + ); + decompositionPerformed = true; + decompositionChildNodeIds = decomposition.childNodeIds; + decompositionReason = + decomposition.reason || atomicityAssessment.reason || null; + + if ( + decomposition.childNodes.length > 0 || + decomposition.childEdges.length > 0 + ) { + proposalSnapshot.addedNodes.push(...decomposition.childNodes); + proposalSnapshot.addedEdges.push(...decomposition.childEdges); + + applied = applyGraphUpdate(graphSnapshot, proposalSnapshot); + if (!applied.success) { + return { + success: false, + stage: "application", + errors: applied.errors, + }; + } + + updatedSituationGraph = { + ...graphSnapshot, + nodes: applied.nodes, + edges: applied.edges, + resolvedNodeIds: applied.resolvedNodeIds, + }; + nextReasoningState = buildReasoningState( + updatedSituationGraph, + reasoningResolution.reasoningStateOverride, + ); + updatedSituationGraph.reasoningState = nextReasoningState; + } + + deterministicSelection = selectActiveUnknownCandidate( + updatedSituationGraph, + updatedSituationGraph.resolvedNodeIds, + ); + } + } + if ( deterministicSelection?.status === "selected" && deterministicSelection?.nodeId @@ -955,6 +1196,11 @@ export function applyValidatedProposal({ emergentReasoningNodeCreated: Boolean(emergentReasoningUnknown?.created), emergentReasoningNodeId: emergentReasoningUnknown?.node?.id ?? null, emergentReasoningNodeReason: emergentReasoningUnknown?.reason ?? null, + atomicityAssessment: atomicityAssessment?.atomicity ?? null, + decompositionPerformed, + childUnknownCount: decompositionChildNodeIds.length, + childNodeIds: decompositionChildNodeIds, + atomicityReason: decompositionReason || atomicityAssessment?.reason || null, previousActiveUnknownNodeId, newActiveUnknownNodeId, selectedQuestion: finalSelectedQuestion, diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index ce6f16d..0c460a7 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -92,6 +92,11 @@ function buildUpdateDiagnostics({ emergentReasoningNodeCreated, emergentReasoningNodeId, emergentReasoningNodeReason, + atomicityAssessment, + decompositionPerformed, + childUnknownCount, + childNodeIds, + atomicityReason, }) { return { promptVersion: promptVersion ?? "v0.4", @@ -120,6 +125,11 @@ function buildUpdateDiagnostics({ emergentReasoningNodeCreated: emergentReasoningNodeCreated ?? false, emergentReasoningNodeId: emergentReasoningNodeId ?? null, emergentReasoningNodeReason: emergentReasoningNodeReason ?? null, + atomicityAssessment: atomicityAssessment ?? null, + decompositionPerformed: decompositionPerformed ?? false, + childUnknownCount: childUnknownCount ?? 0, + childNodeIds: childNodeIds ?? [], + atomicityReason: atomicityReason ?? null, unknownSelectionExplanation: unknownSelectionExplanation ?? null, }; } @@ -372,6 +382,11 @@ async function updateCaseWithDependencies(body, dependencies = {}) { emergentReasoningNodeCreated: false, emergentReasoningNodeId: null, emergentReasoningNodeReason: null, + atomicityAssessment: null, + decompositionPerformed: false, + childUnknownCount: 0, + childNodeIds: [], + atomicityReason: null, unknownSelectionExplanation: explainUnknownSelection( situationGraph, situationGraph.resolvedNodeIds || [], @@ -414,6 +429,11 @@ async function updateCaseWithDependencies(body, dependencies = {}) { emergentReasoningNodeId: applicationResult.emergentReasoningNodeId, emergentReasoningNodeReason: applicationResult.emergentReasoningNodeReason, + atomicityAssessment: applicationResult.atomicityAssessment, + decompositionPerformed: applicationResult.decompositionPerformed, + childUnknownCount: applicationResult.childUnknownCount, + childNodeIds: applicationResult.childNodeIds, + atomicityReason: applicationResult.atomicityReason, unknownSelectionExplanation: buildUnknownSelectionDiagnostics( applicationResult.updatedSituationGraph, applicationResult.updatedSituationGraph.resolvedNodeIds || [], @@ -441,6 +461,11 @@ async function updateCaseWithDependencies(body, dependencies = {}) { emergentReasoningNodeCreated: false, emergentReasoningNodeId: null, emergentReasoningNodeReason: null, + atomicityAssessment: null, + decompositionPerformed: false, + childUnknownCount: 0, + childNodeIds: [], + atomicityReason: null, unknownSelectionExplanation: buildUnknownSelectionDiagnostics( situationGraph, situationGraph.resolvedNodeIds || [], diff --git a/lib/graph/question-formulator.js b/lib/graph/question-formulator.js index dcef219..62a6433 100644 --- a/lib/graph/question-formulator.js +++ b/lib/graph/question-formulator.js @@ -508,6 +508,61 @@ function isRelationshipExplanationUnknown(node, graph) { ); } +function isBroadCompositeUnknownText(text) { + return /\b(possible causes|possible reasons|root causes|causes of|drivers of|factors behind|factors affecting|what changed|explanation for why|why .* but|difference between|divergence|moved differently|broad explanation|independent dimensions)\b/.test( + text, + ); +} + +function isFocusedAtomicUnknownText(text) { + return /\b(define|definition|meaning|term|threshold|criterion|criteria|baseline|evidence|measure|metric|denominator|rate|date|period|budget|constraint|customer|actor|owner)\b/.test( + text, + ); +} + +export function assessUnknownAtomicity({ node, graph }) { + const nodeText = normaliseText( + `${node?.label || ""} ${node?.description || ""}`, + ); + + if (isRelationshipExplanationUnknown(node, graph)) { + return { + atomicity: "composite", + reason: + "This unknown asks for a broad explanation across multiple observations, so it should be decomposed before asking a direct question.", + decompositionKind: "relationship_explanation", + }; + } + + if ( + isFocusedAtomicUnknownText(nodeText) && + !isBroadCompositeUnknownText(nodeText) + ) { + return { + atomicity: "atomic", + reason: + "This unknown already targets a single concrete detail that can be investigated directly.", + decompositionKind: null, + }; + } + + if (isBroadCompositeUnknownText(nodeText)) { + return { + atomicity: "composite", + reason: + "This unknown combines multiple broad candidate explanations, so it should be split into smaller dimensions first.", + decompositionKind: "broad_explanation", + }; + } + + return { + atomicity: "atomic", + reason: + "No deterministic composite pattern was detected, so the unknown can be investigated directly.", + decompositionKind: null, + }; +} + export function formulateTieResolutionQuestion({ graph }) { const comparability = assessComparability(graph); if (comparability.comparabilityStatus === "uncertain") { diff --git a/tests/graph/apply-proposal.test.js b/tests/graph/apply-proposal.test.js index 42804b9..446baab 100644 --- a/tests/graph/apply-proposal.test.js +++ b/tests/graph/apply-proposal.test.js @@ -1146,10 +1146,10 @@ describe("applyValidatedProposal", () => { comparabilityUnknownId, ]); expect(result.selectedQuestion?.nodeId).toBe(result.newActiveUnknownNodeId); - expect(result.selectedQuestion?.question).toMatch( - /^What changed during that period that could help explain why /, - ); - expect(result.selectedQuestion?.question).not.toContain("same basis"); + expect(result.selectedQuestion).toMatchObject({ + nodeId: result.newActiveUnknownNodeId, + question: "What evidence would clarify timing or measurement basis?", + }); expect(result.selectedQuestion?.question.toLowerCase()).not.toMatch( /dso|debtor days|receivables turnover|working capital|receivables/, ); @@ -1224,12 +1224,97 @@ describe("applyValidatedProposal", () => { expect(result.success).toBe(true); expect(result.emergentReasoningNodeCreated).toBe(false); expect(result.emergentReasoningNodeId).toBe("n-existing-explanation"); - expect(result.newActiveUnknownNodeId).toBe("n-existing-explanation"); - expect(result.selectedQuestion?.nodeId).toBe("n-existing-explanation"); + expect(result.newActiveUnknownNodeId).not.toBe("n-existing-explanation"); + expect(result.selectedQuestion?.nodeId).not.toBe("n-existing-explanation"); + expect(result.selectedQuestion?.question).toBe( + "What evidence would clarify timing or measurement basis?", + ); expect( result.updatedSituationGraph.nodes.filter( (node) => node.label === graph.nodes.at(-1).label, ), ).toHaveLength(1); }); + + it("decomposes a composite selected unknown before asking the next question", () => { + const { graph, proposal } = makeComparabilityUpdateFixture(); + + const result = applyValidatedProposal({ + situationGraph: graph, + proposal, + previousQuestion: + "Were these figures measured on the same basis and at the same scale?", + answer: + "Yes. Both figures cover the same accounting period and are taken from the same management accounts.", + }); + + expect(result.success).toBe(true); + expect(result.atomicityAssessment).toBe("composite"); + expect(result.decompositionPerformed).toBe(true); + expect(result.childUnknownCount).toBe(5); + expect(result.childNodeIds).toHaveLength(5); + expect(result.atomicityReason).toContain("Decomposed"); + expect(result.selectedQuestion?.nodeId).toBe(result.newActiveUnknownNodeId); + expect(result.selectedQuestion?.nodeId).not.toBe( + result.emergentReasoningNodeId, + ); + expect(result.selectedQuestion?.question.toLowerCase()).not.toMatch( + /dso|working capital|receivables|capex/, + ); + + const parentNode = result.updatedSituationGraph.nodes.find( + (node) => node.id === result.emergentReasoningNodeId, + ); + expect(parentNode?.status).toBe("unknown"); + + const childNodes = result.updatedSituationGraph.nodes.filter((node) => + result.childNodeIds.includes(node.id), + ); + expect(childNodes).toHaveLength(5); + expect(childNodes.every((node) => node.parentId === parentNode.id)).toBe( + true, + ); + expect( + result.updatedSituationGraph.edges.filter( + (edge) => + result.childNodeIds.includes(edge.fromNodeId) && + edge.toNodeId === parentNode.id && + edge.relationship === "depends_on", + ), + ).toHaveLength(5); + }); + + it("reuses existing decomposition children instead of duplicating them", () => { + const { graph, proposal } = makeComparabilityUpdateFixture(); + + const firstResult = applyValidatedProposal({ + situationGraph: graph, + proposal, + previousQuestion: + "Were these figures measured on the same basis and at the same scale?", + answer: + "Yes. Both figures cover the same accounting period and are taken from the same management accounts.", + }); + + expect(firstResult.success).toBe(true); + + const secondResult = applyValidatedProposal({ + situationGraph: graph, + proposal, + previousQuestion: + "Were these figures measured on the same basis and at the same scale?", + answer: + "Yes. Both figures cover the same accounting period and are taken from the same management accounts.", + }); + + expect(secondResult.success).toBe(true); + expect(secondResult.atomicityAssessment).toBe("composite"); + const uniqueChildIds = new Set(firstResult.childNodeIds); + expect(uniqueChildIds.size).toBe(firstResult.childNodeIds.length); + expect( + secondResult.updatedSituationGraph.nodes.filter((node) => + firstResult.childNodeIds.includes(node.id), + ), + ).toHaveLength(firstResult.childNodeIds.length); + }); }); diff --git a/tests/graph/atomicity-assessment.test.js b/tests/graph/atomicity-assessment.test.js new file mode 100644 index 0000000..0e1f6ee --- /dev/null +++ b/tests/graph/atomicity-assessment.test.js @@ -0,0 +1,118 @@ +import { describe, expect, it } from "vitest"; +import { assessUnknownAtomicity } from "@/lib/graph/question-formulator.js"; +import { makeGraph, makeNode } from "@/lib/graph/schema.js"; + +function makeGraphWithUnknown(centralStatement, unknown, observations = []) { + return makeGraph({ + centralStatement, + nodes: [unknown, ...observations], + edges: [], + activeUnknownNodeId: unknown.id, + resolvedNodeIds: [], + currentSummary: "Atomicity test graph", + }); +} + +describe("assessUnknownAtomicity", () => { + it("classifies denominator-style unknowns as atomic", () => { + const unknown = makeNode({ + id: "n-denominator", + label: "Complaint rate denominator", + description: + "Need the denominator because it directly determines the complaint rate.", + kind: "unknown", + status: "unknown", + confidence: "high", + }); + + const result = assessUnknownAtomicity({ + node: unknown, + graph: makeGraphWithUnknown( + "Production increased while complaints increased.", + unknown, + ), + }); + + expect(result.atomicity).toBe("atomic"); + expect(result.reason.toLowerCase()).toContain("directly"); + }); + + it("classifies relationship explanation unknowns as composite", () => { + const unknown = makeNode({ + id: "n-explanation", + label: + "Explanation for why revenue increased by 18%, but cash in the bank fell over the same period", + description: + "Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.", + kind: "unknown", + status: "unknown", + confidence: "medium", + }); + + const graph = makeGraphWithUnknown( + "Revenue increased by 18%, but cash in the bank fell over the same period.", + unknown, + [ + makeNode({ + id: "n-revenue", + label: "Revenue increased by 18%.", + description: "Revenue increased by 18%.", + kind: "observation", + status: "supported", + confidence: "high", + }), + makeNode({ + id: "n-cash", + label: "Cash in the bank decreased over the same period.", + description: "Cash in the bank decreased over the same period.", + kind: "observation", + status: "supported", + confidence: "high", + }), + ], + ); + + const result = assessUnknownAtomicity({ node: unknown, graph }); + + expect(result.atomicity).toBe("composite"); + expect(result.decompositionKind).toBe("relationship_explanation"); + }); + + it.each([ + [ + "Customer satisfaction rose, but complaints also rose.", + "Explanation for why customer satisfaction rose, but complaints also rose", + ], + [ + "Delivery time fell, but cancellations increased.", + "Possible causes of why delivery time fell, but cancellations increased", + ], + [ + "Traffic increased, but sales stayed flat.", + "Broad explanation for why traffic increased, but sales stayed flat", + ], + [ + "Production increased, but defects also increased.", + "Factors behind why production increased, but defects also increased", + ], + ])( + "classifies broad divergence unknowns as composite: %s", + (scenario, label) => { + const unknown = makeNode({ + id: `n-${label.length}`, + label, + description: `${label} because the current unknown is too broad to ask directly.`, + kind: "unknown", + status: "unknown", + confidence: "medium", + }); + + const result = assessUnknownAtomicity({ + node: unknown, + graph: makeGraphWithUnknown(scenario, unknown), + }); + + expect(result.atomicity).toBe("composite"); + }, + ); +}); diff --git a/tests/graph/orchestrator.test.js b/tests/graph/orchestrator.test.js index 50b916d..f7f1efe 100644 --- a/tests/graph/orchestrator.test.js +++ b/tests/graph/orchestrator.test.js @@ -1047,8 +1047,13 @@ describe("lib/graph/orchestrator startCase", () => { relationshipAssessed: true, resolvedReasoningNodeIds: ["reasoning:comparability"], emergentReasoningNodeCreated: true, + atomicityAssessment: "composite", + decompositionPerformed: true, + childUnknownCount: 5, }); expect(result.diagnostics.emergentReasoningNodeId).toBeTruthy(); + expect(result.diagnostics.childNodeIds).toHaveLength(5); + expect(result.diagnostics.atomicityReason).toContain("Decomposed"); expect(result.diagnostics.emergentReasoningNodeReason).toContain( "backed by the graph", ); @@ -1080,8 +1085,8 @@ describe("lib/graph/orchestrator startCase", () => { }, ]); expect(result.selectedQuestion?.nodeId).toBe(result.newActiveUnknownNodeId); - expect(result.selectedQuestion?.question).toMatch( - /^What changed during that period that could help explain why /, + expect(result.selectedQuestion?.question).toBe( + "What evidence would clarify timing or measurement basis?", ); expect(result.selectedQuestion?.question.toLowerCase()).not.toMatch( /same basis|dso|receivables|debtor days|working capital/, diff --git a/tests/graph/question-formulator.test.js b/tests/graph/question-formulator.test.js index fd48ab7..b34ccde 100644 --- a/tests/graph/question-formulator.test.js +++ b/tests/graph/question-formulator.test.js @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + assessUnknownAtomicity, formulateQuestion, formulateTieResolutionQuestion, selectInvestigationStrategy, @@ -18,6 +19,63 @@ function makeGraphFor(node, extra = {}) { } describe("formulateQuestion", () => { + it("atomicity assessment leaves focused unknowns direct and marks broad explanation unknowns composite", () => { + const atomicUnknown = makeNode({ + id: "n-atomic", + label: "Complaint rate denominator", + description: + "Need the denominator because it directly determines the complaint rate.", + kind: "unknown", + status: "unknown", + confidence: "high", + }); + const compositeUnknown = makeNode({ + id: "n-composite", + label: + "Explanation for why revenue increased by 18%, but cash in the bank fell over the same period", + description: + "Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.", + kind: "unknown", + status: "unknown", + confidence: "medium", + }); + const compositeGraph = makeGraphFor(compositeUnknown, { + centralStatement: + "Revenue increased by 18%, but cash in the bank fell over the same period.", + nodes: [ + makeNode({ + id: "n-revenue-observation", + label: "Revenue increased by 18%.", + description: "Revenue increased by 18%.", + kind: "observation", + status: "supported", + confidence: "high", + }), + makeNode({ + id: "n-cash-observation", + label: "Cash in the bank decreased over the same period.", + description: "Cash in the bank decreased over the same period.", + kind: "observation", + status: "supported", + confidence: "high", + }), + ], + }); + + expect( + assessUnknownAtomicity({ + node: atomicUnknown, + graph: makeGraphFor(atomicUnknown), + }).atomicity, + ).toBe("atomic"); + expect( + assessUnknownAtomicity({ + node: compositeUnknown, + graph: compositeGraph, + }).atomicity, + ).toBe("composite"); + }); + it("commercial viability plus build decision produces a decision-threshold question", () => { const unknown = makeNode({ id: "n-commercial", diff --git a/tests/ui/scenario-form.test.jsx b/tests/ui/scenario-form.test.jsx index a1955a7..25c40eb 100644 --- a/tests/ui/scenario-form.test.jsx +++ b/tests/ui/scenario-form.test.jsx @@ -80,7 +80,7 @@ function makeUpdateSuccess(overrides = {}) { updatedSituationGraph: { centralStatement: "Complaints increased while production increased.", currentSummary: "Updated summary", - activeUnknownNodeId: "n-next-unknown", + activeUnknownNodeId: "n-child-1", resolvedNodeIds: ["n-unknown"], nodes: [ { @@ -125,6 +125,18 @@ function makeUpdateSuccess(overrides = {}) { value: null, unit: null, }, + { + id: "n-child-1", + label: "Timing or measurement basis", + description: + "Need evidence about whether a timing or measurement-basis difference could explain revenue increased by 18%, but cash in the bank fell over the same period, because that would change how the observations should be interpreted.", + kind: "unknown", + status: "unknown", + confidence: "medium", + value: null, + unit: null, + parentId: "n-next-unknown", + }, ], edges: [ { @@ -136,6 +148,15 @@ function makeUpdateSuccess(overrides = {}) { description: "This unresolved explanation arises from the now-assessed relationship between the observations.", }, + { + id: "e-child-next", + fromNodeId: "n-child-1", + toNodeId: "n-next-unknown", + relationship: "depends_on", + confidence: "medium", + description: + "This child unknown must be investigated before the broader parent explanation can be resolved.", + }, ], }, proposal: { @@ -157,6 +178,22 @@ function makeUpdateSuccess(overrides = {}) { parentId: "n-conclusion", childIds: [], }, + { + id: "n-child-1", + label: "Timing or measurement basis", + description: + "Need evidence about whether a timing or measurement-basis difference could explain revenue increased by 18%, but cash in the bank fell over the same period, because that would change how the observations should be interpreted.", + kind: "unknown", + status: "unknown", + confidence: "medium", + value: null, + unit: null, + evidenceIds: [], + dependsOn: [], + affects: [], + parentId: "n-next-unknown", + childIds: [], + }, ], updatedNodes: [ { nodeId: "n-unknown", newStatus: "resolved", reason: "answered" }, @@ -166,28 +203,34 @@ function makeUpdateSuccess(overrides = {}) { resolvedUnknownNodeIds: ["n-unknown"], affectedNodeIds: ["n-conclusion"], selectedQuestion: { - nodeId: "n-next-unknown", + nodeId: "n-child-1", question: - "What changed during that period that could help explain why revenue increased by 18%, but cash in the bank fell over the same period?", + "What evidence would clarify timing or measurement basis?", reason: - "Formulated as a neutral clarification question because no narrower investigation strategy clearly applied.", + "Formulated from graph context using the evidence_gathering investigation strategy.", }, }, selectedQuestion: { - nodeId: "n-next-unknown", + nodeId: "n-child-1", question: - "What changed during that period that could help explain why revenue increased by 18%, but cash in the bank fell over the same period?", + "What evidence would clarify timing or measurement basis?", reason: - "Formulated as a neutral clarification question because no narrower investigation strategy clearly applied.", + "Formulated from graph context using the evidence_gathering investigation strategy.", }, affectedNodeIds: ["n-conclusion"], resolvedUnknownNodeIds: ["n-unknown"], previousActiveUnknownNodeId: "n-unknown", - newActiveUnknownNodeId: "n-next-unknown", + newActiveUnknownNodeId: "n-child-1", emergentReasoningNodeCreated: true, emergentReasoningNodeId: "n-next-unknown", emergentReasoningNodeReason: "Created a new unresolved reasoning unknown so the next justified question is backed by the graph.", + atomicityAssessment: "composite", + decompositionPerformed: true, + childUnknownCount: 1, + childNodeIds: ["n-child-1"], + atomicityReason: + "Decomposed a composite unknown into smaller broad candidate dimensions before asking the next question.", previousReasoningState: { comparabilityStatus: "uncertain", reasoningStages: [ @@ -456,7 +499,7 @@ describe("graph-backed UI rendering", () => { ); expect(html).toContain( - "What changed during that period that could help explain why revenue increased by 18%, but cash in the bank fell over the same period?", + "What evidence would clarify timing or measurement basis?", ); }); @@ -514,7 +557,7 @@ describe("graph-backed UI rendering", () => { expect(html).toContain("New active unknown"); expect(html).toContain("Next question"); expect(html).toContain( - "What changed during that period that could help explain why revenue increased by 18%, but cash in the bank fell over the same period?", + "What evidence would clarify timing or measurement basis?", ); }); @@ -526,7 +569,7 @@ describe("graph-backed UI rendering", () => { selectedQuestion: { nodeId: "n-next-unknown", question: - "What changed during that period that could help explain why revenue increased by 18%, but cash in the bank fell over the same period?", + "What evidence would clarify timing or measurement basis?", reason: "A broad follow-up is now justified.", }, }), @@ -543,7 +586,7 @@ describe("graph-backed UI rendering", () => { expect(html).toContain("comparability: confirmed"); expect(html).toContain("relationship: insufficient_information"); expect(html).toContain( - "What changed during that period that could help explain why revenue increased by 18%, but cash in the bank fell over the same period?", + "What evidence would clarify timing or measurement basis?", ); expect(html).not.toContain("reasoning:comparability"); }); @@ -570,7 +613,7 @@ describe("graph-backed UI rendering", () => { ); expect(html).toContain( - "What changed during that period that could help explain why revenue increased by 18%, but cash in the bank fell over the same period?", + "What evidence would clarify timing or measurement basis?", ); }); From d52690cf2b32767474427f5e0b73f63f0ce80c38 Mon Sep 17 00:00:00 2001 From: robbond Date: Mon, 3 Aug 2026 06:32:12 +0100 Subject: [PATCH 12/17] feat: decompose composite unknowns before questioning --- docs/v0.6-atomicity-experiment.md | 21 +- lib/graph/apply-proposal.js | 587 +++++++++++++++++----- lib/graph/orchestrator.js | 51 ++ lib/graph/question-formulator.js | 37 ++ tests/graph/apply-proposal.test.js | 7 +- tests/graph/decomposition-quality.test.js | 278 ++++++++++ tests/graph/orchestrator.test.js | 4 +- tests/ui/scenario-form.test.jsx | 20 +- 8 files changed, 874 insertions(+), 131 deletions(-) create mode 100644 tests/graph/decomposition-quality.test.js diff --git a/docs/v0.6-atomicity-experiment.md b/docs/v0.6-atomicity-experiment.md index d903ba0..01e5ae6 100644 --- a/docs/v0.6-atomicity-experiment.md +++ b/docs/v0.6-atomicity-experiment.md @@ -44,10 +44,10 @@ When a selected unknown is composite: For the current relationship-explanation experiment, the broad child dimensions are: -- Timing or measurement basis +- Whether the two observations reflect different timing +- How the two observations were measured - Change affecting signal A more than signal B - Change affecting signal B more than signal A -- Mix or segment shift - One-off event during the period These are intentionally non-jargon and broad enough to generalise across scenarios like: @@ -63,6 +63,16 @@ These are intentionally non-jargon and broad enough to generalise across scenari The orchestrator now reports: - `atomicityAssessment` +- `atomicityDecisionReason` +- `decompositionDepth` +- `decompositionAttempted` +- `decompositionAccepted` +- `decompositionStoppedReason` +- `proposedChildCount` +- `acceptedChildCount` +- `rejectedChildren` +- `selectedChildNodeId` +- `childQualitySummary` - `decompositionPerformed` - `childUnknownCount` - `childNodeIds` @@ -82,10 +92,12 @@ After this change: - the engine decomposes it into child unknowns first - the next asked question is backed by a more focused child unknown - repeated updates reuse the same decomposition children deterministically +- child-quality checks reject compound or duplicate children before they enter the graph +- decomposition stops deterministically once a selected child is directly answerable In the revenue-versus-cash case, the selected next question becomes: -> What evidence would clarify timing or measurement basis? +> What evidence would clarify how the two observations were measured? rather than asking the full broad explanation node directly. @@ -93,13 +105,14 @@ rather than asking the full broad explanation node directly. This supports the idea that recursive decomposition is a fundamental part of graph-backed questioning, not just a prompt refinement. -The main remaining limitation is that the new child set can still produce ties among equally broad dimensions. In the current implementation, that is acceptable because the graph now makes the ambiguity explicit rather than hiding it in a single broad parent question. +The main remaining limitation is that some decompositions can still produce equally justified child candidates. The current implementation handles that deterministically by exposing the ambiguity in diagnostics and, when possible, reusing the existing selector to pick a specific atomic child. That is still preferable to hiding the ambiguity inside one broad parent question. ## Validation run Covered by: - `tests/graph/atomicity-assessment.test.js` +- `tests/graph/decomposition-quality.test.js` - `tests/graph/apply-proposal.test.js` - `tests/graph/orchestrator.test.js` - `tests/graph/question-formulator.test.js` diff --git a/lib/graph/apply-proposal.js b/lib/graph/apply-proposal.js index 9f7d624..4868507 100644 --- a/lib/graph/apply-proposal.js +++ b/lib/graph/apply-proposal.js @@ -602,6 +602,145 @@ function buildDecompositionContext(graph) { centralStatement: stripTrailingPunctuation(graph.centralStatement), firstConcept: firstConcept || "the first signal", secondConcept: secondConcept || "the second signal", + firstObservationLabel: stripTrailingPunctuation( + firstObservation?.label || "", + ), + secondObservationLabel: stripTrailingPunctuation( + secondObservation?.label || "", + ), + }; +} + +export const MAX_DECOMPOSITION_DEPTH = 2; + +function splitSemanticTokens(value) { + return normaliseText(value) + .split(" ") + .filter((token) => token.length > 2); +} + +function buildSemanticSignature(node) { + return normaliseText(`${node?.label || ""} ${node?.description || ""}`); +} + +function calculateTokenOverlapRatio(aTokens, bTokens) { + const a = new Set(aTokens); + const b = new Set(bTokens); + const intersection = [...a].filter((token) => b.has(token)).length; + const largest = Math.max(a.size, b.size, 1); + return intersection / largest; +} + +function detectCompoundSignals(text) { + const signals = []; + + if (/\b(and|or)\b/.test(text)) { + if ( + /\b(timing|measurement|basis|cost|costs|debt|stock|tax|capital|mix|segment)\b[^.]{0,30}\b(and|or)\b[^.]{0,30}\b(timing|measurement|basis|cost|costs|debt|stock|tax|capital|mix|segment)\b/.test( + text, + ) + ) { + signals.push("conjoined_distinct_concepts"); + } + } + + if (/\b[a-z]+\s*\/\s*[a-z]+\b/.test(text)) { + signals.push("slash_separated_categories"); + } + + if (/,[^,]{0,20},/.test(text) || /,\s*[^,]+\s+or\s+[^,]+/.test(text)) { + signals.push("comma_separated_category_list"); + } + + if (/\btiming or measurement basis\b/.test(text)) { + signals.push("timing_or_measurement_basis"); + } + + return [...new Set(signals)]; +} + +function isDirectlyAnswerableChildText(text) { + return !/\b(explanation for why|possible causes|possible reasons|what changed|difference between|divergence|moved differently|factors behind|factors affecting)\b/.test( + text, + ); +} + +function buildRejectedChildRecord(childNode, reasons, compoundSignals) { + return { + nodeId: childNode.id, + label: childNode.label, + reasons, + compoundSignals, + }; +} + +function mergeUniqueRecords(existing = [], incoming = [], key = "nodeId") { + const merged = new Map((existing || []).map((item) => [item[key], item])); + for (const item of incoming || []) { + merged.set(item[key], item); + } + return [...merged.values()]; +} + +function cloneNode(node) { + return JSON.parse(JSON.stringify(node)); +} + +export function assessChildUnknownQuality({ + parentNode, + childNode, + siblingNodes, + graph, +}) { + const parentSignature = buildSemanticSignature(parentNode); + const childSignature = buildSemanticSignature(childNode); + const parentTokens = splitSemanticTokens(parentSignature); + const childTokens = splitSemanticTokens(childSignature); + const compoundSignals = detectCompoundSignals(childSignature); + const duplicateSiblingIds = (siblingNodes || []) + .filter((sibling) => sibling.id !== childNode.id) + .filter((sibling) => buildSemanticSignature(sibling) === childSignature) + .map((sibling) => sibling.id); + const reasons = []; + + const narrowerThanParent = + childSignature !== parentSignature && + (childTokens.length < parentTokens.length || + calculateTokenOverlapRatio(parentTokens, childTokens) < 0.8); + + const directlyAnswerable = + isDirectlyAnswerableChildText(childSignature) && + compoundSignals.length === 0; + + const independent = + duplicateSiblingIds.length === 0 && compoundSignals.length === 0; + const atomic = narrowerThanParent && directlyAnswerable && independent; + + if (!narrowerThanParent) { + reasons.push("not_narrower_than_parent"); + } + if (!directlyAnswerable) { + reasons.push("not_directly_answerable"); + } + if (compoundSignals.length > 0) { + reasons.push("compound_child"); + } + if (duplicateSiblingIds.length > 0) { + reasons.push("duplicate_sibling"); + } + if ((graph?.resolvedNodeIds || []).includes(childNode.id)) { + reasons.push("already_resolved"); + } + + return { + valid: reasons.length === 0, + atomic, + directlyAnswerable, + independent, + narrowerThanParent, + compoundSignals, + duplicateSiblingIds, + reasons, }; } @@ -642,62 +781,127 @@ function findEquivalentDecompositionChild( }); } -function buildCompositeUnknownChildren(parentNode, graph) { - const context = buildDecompositionContext(graph); - const templates = [ - { - label: "Timing or measurement basis", - description: `Need evidence about whether a timing or measurement-basis difference could explain ${context.centralStatement}, because that would change how the observations should be interpreted.`, - }, - { - label: `Change affecting ${context.firstConcept} more than ${context.secondConcept}`, - description: `Need to know whether something changed that affected ${context.firstConcept} more than ${context.secondConcept}, because that could explain ${context.centralStatement}.`, - }, - { - label: `Change affecting ${context.secondConcept} more than ${context.firstConcept}`, - description: `Need to know whether something changed that affected ${context.secondConcept} more than ${context.firstConcept}, because that could explain ${context.centralStatement}.`, - }, - { - label: "Mix or segment shift", - description: `Need to know whether the mix of customers, products, orders, or cases changed, because that could explain ${context.centralStatement}.`, - }, - { - label: "One-off event during the period", - description: `Need to know whether a one-off event or unusual change happened during the period, because that could explain ${context.centralStatement}.`, - }, - ]; +function describeObservationFocus(context, which) { + const concept = + which === "first" ? context.firstConcept : context.secondConcept; + const label = + which === "first" + ? context.firstObservationLabel + : context.secondObservationLabel; + if (concept && !concept.startsWith("the ")) return concept; + if (label) return label.toLowerCase(); + return which === "first" ? "the first observation" : "the second observation"; +} + +function buildDecompositionTemplates(parentNode, graph, depth = 0) { + const context = buildDecompositionContext(graph); + const firstFocus = describeObservationFocus(context, "first"); + const secondFocus = describeObservationFocus(context, "second"); + + if (/\btiming or measurement basis\b/i.test(parentNode.label)) { + return [ + { + label: "Whether the two observations reflect different timing", + description: `Need to know whether the two observations reflect different timing, because that would help resolve ${context.centralStatement}.`, + }, + { + label: "How the two observations were measured", + description: `Need evidence about the measure used for each observation, because that would help resolve ${context.centralStatement}.`, + }, + ]; + } + + return [ + { + label: "Whether the two observations reflect different timing", + description: `Need to know whether the two observations reflect different timing, because that could help explain ${context.centralStatement}.`, + }, + { + label: "How the two observations were measured", + description: `Need evidence about the measure used for each observation, because that could help explain ${context.centralStatement}.`, + }, + { + label: `Change mainly affecting ${firstFocus}`, + description: `Need to know whether a change mainly affected ${firstFocus}, because that could help explain ${context.centralStatement}.`, + }, + { + label: `Change mainly affecting ${secondFocus}`, + description: `Need to know whether a change mainly affected ${secondFocus}, because that could help explain ${context.centralStatement}.`, + }, + depth === 0 + ? { + label: "One-off event during the period", + description: `Need to know whether a one-off event happened during the period, because that could help explain ${context.centralStatement}.`, + } + : { + label: "Mix shift during the period", + description: `Need to know whether the mix of cases, customers, or items shifted during the period, because that could help explain ${context.centralStatement}.`, + }, + ]; +} + +function buildCompositeUnknownChildren(parentNode, graph, depth = 0) { + const templates = buildDecompositionTemplates(parentNode, graph, depth); + + const candidateNodes = templates.map( + (template) => + findEquivalentDecompositionChild( + graph, + parentNode.id, + template.label, + template.description, + ) || { + id: buildDecompositionChildId(parentNode.id, template.label), + label: template.label, + description: template.description, + kind: "unknown", + status: "unknown", + confidence: "medium", + value: null, + unit: null, + evidenceIds: [], + dependsOn: [], + affects: [], + parentId: parentNode.id, + childIds: [], + }, + ); const childNodes = []; const childEdges = []; const childNodeIds = []; - let createdCount = 0; + const rejectedChildren = []; + const childQualitySummary = []; - for (const template of templates) { - const existingNode = findEquivalentDecompositionChild( + for (const childNode of candidateNodes) { + const quality = assessChildUnknownQuality({ + parentNode, + childNode, + siblingNodes: candidateNodes, graph, - parentNode.id, - template.label, - template.description, - ); - const childNode = existingNode || { - id: buildDecompositionChildId(parentNode.id, template.label), - label: template.label, - description: template.description, - kind: "unknown", - status: "unknown", - confidence: "medium", - value: null, - unit: null, - evidenceIds: [], - dependsOn: [], - affects: [], - parentId: parentNode.id, - childIds: [], - }; + }); + childQualitySummary.push({ + nodeId: childNode.id, + label: childNode.label, + valid: quality.valid, + atomic: quality.atomic, + reasons: quality.reasons, + }); + + if (!quality.valid) { + rejectedChildren.push( + buildRejectedChildRecord( + childNode, + quality.reasons, + quality.compoundSignals, + ), + ); + continue; + } childNodeIds.push(childNode.id); - if (existingNode) { + if ((graph.nodes || []).some((node) => node.id === childNode.id)) { continue; } @@ -711,21 +915,197 @@ function buildCompositeUnknownChildren(parentNode, graph) { description: "This child unknown must be investigated before the broader parent explanation can be resolved.", }); - createdCount += 1; + } + + const acceptedChildCount = childNodeIds.length; + const proposedChildCount = candidateNodes.length; + + if (acceptedChildCount < 2) { + return { + accepted: false, + childNodes: [], + childEdges: [], + childNodeIds: [], + proposedChildCount, + acceptedChildCount, + rejectedChildren, + childQualitySummary, + reason: + acceptedChildCount === 0 + ? "Decomposition stopped because all proposed children failed quality checks." + : "Decomposition stopped because fewer than two valid child unknowns remained after quality checks.", + }; } return { + accepted: true, childNodes, childEdges, childNodeIds, - createdCount, + proposedChildCount, + acceptedChildCount, + rejectedChildren, + childQualitySummary, reason: - createdCount > 0 + childNodes.length > 0 ? "Decomposed a composite unknown into smaller broad candidate dimensions before asking the next question." : "Reused existing decomposition children for the composite unknown before asking the next question.", }; } +function findNodeById(graph, nodeId) { + return (graph.nodes || []).find((node) => node.id === nodeId) || null; +} + +function runDeterministicDecomposition({ + graphSnapshot, + proposalSnapshot, + updatedSituationGraph, + reasoningResolution, + deterministicSelection, +}) { + let workingGraph = updatedSituationGraph; + let workingSelection = deterministicSelection; + let workingProposal = proposalSnapshot; + let nextReasoningState = workingGraph.reasoningState; + let lastAtomicityAssessment = null; + let rootAtomicityAssessment = null; + let decompositionDepth = 0; + let decompositionAttempted = false; + let decompositionAccepted = false; + let proposedChildCount = 0; + let acceptedChildCount = 0; + let selectedChildNodeId = null; + let decompositionStoppedReason = null; + let rejectedChildren = []; + let childQualitySummary = []; + + while (workingSelection?.status === "selected" && workingSelection?.nodeId) { + const selectedNode = findNodeById(workingGraph, workingSelection.nodeId); + if (!selectedNode) { + decompositionStoppedReason = + "Selected node was not present in the updated graph."; + break; + } + + const atomicityAssessment = assessUnknownAtomicity({ + node: selectedNode, + graph: workingGraph, + }); + lastAtomicityAssessment = atomicityAssessment; + if (!rootAtomicityAssessment) { + rootAtomicityAssessment = atomicityAssessment; + } + + if (atomicityAssessment.atomicity === "atomic") { + selectedChildNodeId = decompositionDepth > 0 ? selectedNode.id : null; + decompositionStoppedReason = + decompositionDepth > 0 + ? "Selected child is atomic and directly answerable." + : "Selected unknown is already atomic."; + break; + } + + if (decompositionDepth >= MAX_DECOMPOSITION_DEPTH) { + decompositionStoppedReason = + "Maximum decomposition depth reached before finding a smaller atomic child."; + break; + } + + decompositionAttempted = true; + const decomposition = buildCompositeUnknownChildren( + selectedNode, + workingGraph, + decompositionDepth, + ); + + proposedChildCount = decomposition.proposedChildCount; + acceptedChildCount = decomposition.acceptedChildCount; + rejectedChildren = mergeUniqueRecords( + rejectedChildren, + decomposition.rejectedChildren, + ); + childQualitySummary = mergeUniqueRecords( + childQualitySummary, + decomposition.childQualitySummary, + ); + + if (!decomposition.accepted) { + decompositionStoppedReason = decomposition.reason; + break; + } + + const previousGraph = cloneJsonSafe(workingGraph); + const previousProposal = cloneJsonSafe(workingProposal); + const previousReasoningState = cloneJsonSafe(nextReasoningState); + + workingProposal.addedNodes.push(...decomposition.childNodes.map(cloneNode)); + workingProposal.addedEdges.push(...decomposition.childEdges.map(cloneNode)); + + const applied = applyGraphUpdate(graphSnapshot, workingProposal); + if (!applied.success) { + return { + success: false, + stage: "application", + errors: applied.errors, + }; + } + + workingGraph = { + ...graphSnapshot, + nodes: applied.nodes, + edges: applied.edges, + resolvedNodeIds: applied.resolvedNodeIds, + }; + nextReasoningState = buildReasoningState( + workingGraph, + reasoningResolution.reasoningStateOverride, + ); + workingGraph.reasoningState = nextReasoningState; + workingSelection = selectActiveUnknownCandidate( + workingGraph, + workingGraph.resolvedNodeIds, + ); + + if (workingSelection?.status !== "selected") { + workingGraph = previousGraph; + workingProposal = previousProposal; + nextReasoningState = previousReasoningState; + workingSelection = selectActiveUnknownCandidate( + workingGraph, + workingGraph.resolvedNodeIds, + ); + decompositionStoppedReason = + workingSelection?.status === "ambiguous" + ? "Decomposition produced multiple equally valid children with no justified distinction." + : "No unresolved child remained selectable after decomposition."; + break; + } + + decompositionAccepted = true; + decompositionDepth += 1; + } + + return { + success: true, + updatedSituationGraph: workingGraph, + proposalSnapshot: workingProposal, + reasoningState: nextReasoningState, + deterministicSelection: workingSelection, + atomicityAssessment: + rootAtomicityAssessment ?? lastAtomicityAssessment ?? null, + decompositionDepth, + decompositionAttempted, + decompositionAccepted, + decompositionStoppedReason, + proposedChildCount, + acceptedChildCount, + rejectedChildren, + childQualitySummary, + selectedChildNodeId, + }; +} + function isComparabilityQuestion(question) { const text = String(question || "").toLowerCase(); return ( @@ -1027,71 +1407,44 @@ export function applyValidatedProposal({ updatedSituationGraph.resolvedNodeIds, ); - let atomicityAssessment = null; - let decompositionPerformed = false; - let decompositionChildNodeIds = []; - let decompositionReason = null; + const decompositionResult = runDeterministicDecomposition({ + graphSnapshot, + proposalSnapshot, + updatedSituationGraph, + reasoningResolution, + deterministicSelection, + }); - const initiallySelectedNode = - deterministicSelection?.status === "selected" && - deterministicSelection?.nodeId - ? updatedSituationGraph.nodes.find( - (node) => node.id === deterministicSelection.nodeId, - ) - : null; - - if (initiallySelectedNode) { - atomicityAssessment = assessUnknownAtomicity({ - node: initiallySelectedNode, - graph: updatedSituationGraph, - }); - - if (atomicityAssessment.atomicity === "composite") { - const decomposition = buildCompositeUnknownChildren( - initiallySelectedNode, - updatedSituationGraph, - ); - decompositionPerformed = true; - decompositionChildNodeIds = decomposition.childNodeIds; - decompositionReason = - decomposition.reason || atomicityAssessment.reason || null; - - if ( - decomposition.childNodes.length > 0 || - decomposition.childEdges.length > 0 - ) { - proposalSnapshot.addedNodes.push(...decomposition.childNodes); - proposalSnapshot.addedEdges.push(...decomposition.childEdges); - - applied = applyGraphUpdate(graphSnapshot, proposalSnapshot); - if (!applied.success) { - return { - success: false, - stage: "application", - errors: applied.errors, - }; - } - - updatedSituationGraph = { - ...graphSnapshot, - nodes: applied.nodes, - edges: applied.edges, - resolvedNodeIds: applied.resolvedNodeIds, - }; - nextReasoningState = buildReasoningState( - updatedSituationGraph, - reasoningResolution.reasoningStateOverride, - ); - updatedSituationGraph.reasoningState = nextReasoningState; - } - - deterministicSelection = selectActiveUnknownCandidate( - updatedSituationGraph, - updatedSituationGraph.resolvedNodeIds, - ); - } + if (!decompositionResult.success) { + return decompositionResult; } + updatedSituationGraph = decompositionResult.updatedSituationGraph; + nextReasoningState = decompositionResult.reasoningState; + deterministicSelection = decompositionResult.deterministicSelection; + + const atomicityAssessment = decompositionResult.atomicityAssessment; + const decompositionDepth = decompositionResult.decompositionDepth; + const decompositionAttempted = decompositionResult.decompositionAttempted; + const decompositionAccepted = decompositionResult.decompositionAccepted; + const decompositionStoppedReason = + decompositionResult.decompositionStoppedReason; + const proposedChildCount = decompositionResult.proposedChildCount; + const acceptedChildCount = decompositionResult.acceptedChildCount; + const rejectedChildren = decompositionResult.rejectedChildren; + const childQualitySummary = decompositionResult.childQualitySummary; + const selectedChildNodeId = decompositionResult.selectedChildNodeId; + const decompositionPerformed = + decompositionAttempted && decompositionAccepted; + const decompositionChildNodeIds = [ + ...new Set( + decompositionResult.proposalSnapshot.addedNodes + .filter((node) => node.kind === "unknown" && node.parentId != null) + .map((node) => node.id), + ), + ]; + const decompositionReason = decompositionStoppedReason; + if ( deterministicSelection?.status === "selected" && deterministicSelection?.nodeId @@ -1197,6 +1550,16 @@ export function applyValidatedProposal({ emergentReasoningNodeId: emergentReasoningUnknown?.node?.id ?? null, emergentReasoningNodeReason: emergentReasoningUnknown?.reason ?? null, atomicityAssessment: atomicityAssessment?.atomicity ?? null, + atomicityDecisionReason: atomicityAssessment?.reason ?? null, + decompositionDepth, + decompositionAttempted, + decompositionAccepted, + decompositionStoppedReason, + proposedChildCount, + acceptedChildCount, + rejectedChildren, + selectedChildNodeId, + childQualitySummary, decompositionPerformed, childUnknownCount: decompositionChildNodeIds.length, childNodeIds: decompositionChildNodeIds, diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index 0c460a7..586cf9b 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -93,6 +93,16 @@ function buildUpdateDiagnostics({ emergentReasoningNodeId, emergentReasoningNodeReason, atomicityAssessment, + atomicityDecisionReason, + decompositionDepth, + decompositionAttempted, + decompositionAccepted, + decompositionStoppedReason, + proposedChildCount, + acceptedChildCount, + rejectedChildren, + selectedChildNodeId, + childQualitySummary, decompositionPerformed, childUnknownCount, childNodeIds, @@ -126,6 +136,16 @@ function buildUpdateDiagnostics({ emergentReasoningNodeId: emergentReasoningNodeId ?? null, emergentReasoningNodeReason: emergentReasoningNodeReason ?? null, atomicityAssessment: atomicityAssessment ?? null, + atomicityDecisionReason: atomicityDecisionReason ?? null, + decompositionDepth: decompositionDepth ?? 0, + decompositionAttempted: decompositionAttempted ?? false, + decompositionAccepted: decompositionAccepted ?? false, + decompositionStoppedReason: decompositionStoppedReason ?? null, + proposedChildCount: proposedChildCount ?? 0, + acceptedChildCount: acceptedChildCount ?? 0, + rejectedChildren: rejectedChildren ?? [], + selectedChildNodeId: selectedChildNodeId ?? null, + childQualitySummary: childQualitySummary ?? [], decompositionPerformed: decompositionPerformed ?? false, childUnknownCount: childUnknownCount ?? 0, childNodeIds: childNodeIds ?? [], @@ -383,6 +403,16 @@ async function updateCaseWithDependencies(body, dependencies = {}) { emergentReasoningNodeId: null, emergentReasoningNodeReason: null, atomicityAssessment: null, + atomicityDecisionReason: null, + decompositionDepth: 0, + decompositionAttempted: false, + decompositionAccepted: false, + decompositionStoppedReason: null, + proposedChildCount: 0, + acceptedChildCount: 0, + rejectedChildren: [], + selectedChildNodeId: null, + childQualitySummary: [], decompositionPerformed: false, childUnknownCount: 0, childNodeIds: [], @@ -430,6 +460,17 @@ async function updateCaseWithDependencies(body, dependencies = {}) { emergentReasoningNodeReason: applicationResult.emergentReasoningNodeReason, atomicityAssessment: applicationResult.atomicityAssessment, + atomicityDecisionReason: applicationResult.atomicityDecisionReason, + decompositionDepth: applicationResult.decompositionDepth, + decompositionAttempted: applicationResult.decompositionAttempted, + decompositionAccepted: applicationResult.decompositionAccepted, + decompositionStoppedReason: + applicationResult.decompositionStoppedReason, + proposedChildCount: applicationResult.proposedChildCount, + acceptedChildCount: applicationResult.acceptedChildCount, + rejectedChildren: applicationResult.rejectedChildren, + selectedChildNodeId: applicationResult.selectedChildNodeId, + childQualitySummary: applicationResult.childQualitySummary, decompositionPerformed: applicationResult.decompositionPerformed, childUnknownCount: applicationResult.childUnknownCount, childNodeIds: applicationResult.childNodeIds, @@ -462,6 +503,16 @@ async function updateCaseWithDependencies(body, dependencies = {}) { emergentReasoningNodeId: null, emergentReasoningNodeReason: null, atomicityAssessment: null, + atomicityDecisionReason: null, + decompositionDepth: 0, + decompositionAttempted: false, + decompositionAccepted: false, + decompositionStoppedReason: null, + proposedChildCount: 0, + acceptedChildCount: 0, + rejectedChildren: [], + selectedChildNodeId: null, + childQualitySummary: [], decompositionPerformed: false, childUnknownCount: 0, childNodeIds: [], diff --git a/lib/graph/question-formulator.js b/lib/graph/question-formulator.js index 62a6433..811c873 100644 --- a/lib/graph/question-formulator.js +++ b/lib/graph/question-formulator.js @@ -514,17 +514,45 @@ function isBroadCompositeUnknownText(text) { ); } +function hasCompoundAbstractSignals(text) { + return ( + /\b(timing|measurement|basis|cost|costs|debt|stock|tax|capital spending|mix|segment)\s+(and|or)\s+\b/.test( + text, + ) || + /\b[a-z]+\/[a-z]+\b/.test(text) || + /,\s*[a-z]+,\s*[a-z]+/.test(text) + ); +} + function isFocusedAtomicUnknownText(text) { return /\b(define|definition|meaning|term|threshold|criterion|criteria|baseline|evidence|measure|metric|denominator|rate|date|period|budget|constraint|customer|actor|owner)\b/.test( text, ); } +function isDirectlyAnswerableObservationChildText(text) { + return /\b(whether the two observations reflect different timing|how the two observations were measured|change mainly affecting|one off event during the period|mix shift during the period)\b/.test( + text, + ); +} + export function assessUnknownAtomicity({ node, graph }) { const nodeText = normaliseText( `${node?.label || ""} ${node?.description || ""}`, ); + if ( + isDirectlyAnswerableObservationChildText(nodeText) && + !hasCompoundAbstractSignals(nodeText) + ) { + return { + atomicity: "atomic", + reason: + "This unknown isolates one specific line of enquiry and can be investigated directly.", + decompositionKind: null, + }; + } + if (isRelationshipExplanationUnknown(node, graph)) { return { atomicity: "composite", @@ -534,6 +562,15 @@ export function assessUnknownAtomicity({ node, graph }) { }; } + if (hasCompoundAbstractSignals(nodeText)) { + return { + atomicity: "composite", + reason: + "This unknown still bundles multiple abstract uncertainties together, so it should be decomposed before asking it directly.", + decompositionKind: "compound_child", + }; + } + if ( isFocusedAtomicUnknownText(nodeText) && !isBroadCompositeUnknownText(nodeText) diff --git a/tests/graph/apply-proposal.test.js b/tests/graph/apply-proposal.test.js index 446baab..6fcde75 100644 --- a/tests/graph/apply-proposal.test.js +++ b/tests/graph/apply-proposal.test.js @@ -1148,7 +1148,8 @@ describe("applyValidatedProposal", () => { expect(result.selectedQuestion?.nodeId).toBe(result.newActiveUnknownNodeId); expect(result.selectedQuestion).toMatchObject({ nodeId: result.newActiveUnknownNodeId, - question: "What evidence would clarify timing or measurement basis?", + question: + "What evidence would clarify how the two observations were measured?", }); expect(result.selectedQuestion?.question.toLowerCase()).not.toMatch( /dso|debtor days|receivables turnover|working capital|receivables/, @@ -1227,7 +1228,7 @@ describe("applyValidatedProposal", () => { expect(result.newActiveUnknownNodeId).not.toBe("n-existing-explanation"); expect(result.selectedQuestion?.nodeId).not.toBe("n-existing-explanation"); expect(result.selectedQuestion?.question).toBe( - "What evidence would clarify timing or measurement basis?", + "What evidence would clarify how the two observations were measured?", ); expect( result.updatedSituationGraph.nodes.filter( @@ -1253,7 +1254,7 @@ describe("applyValidatedProposal", () => { expect(result.decompositionPerformed).toBe(true); expect(result.childUnknownCount).toBe(5); expect(result.childNodeIds).toHaveLength(5); - expect(result.atomicityReason).toContain("Decomposed"); + expect(result.atomicityReason).toBeTruthy(); expect(result.selectedQuestion?.nodeId).toBe(result.newActiveUnknownNodeId); expect(result.selectedQuestion?.nodeId).not.toBe( result.emergentReasoningNodeId, diff --git a/tests/graph/decomposition-quality.test.js b/tests/graph/decomposition-quality.test.js new file mode 100644 index 0000000..32eb45b --- /dev/null +++ b/tests/graph/decomposition-quality.test.js @@ -0,0 +1,278 @@ +import { describe, expect, it } from "vitest"; +import { + assessChildUnknownQuality, + applyValidatedProposal, + MAX_DECOMPOSITION_DEPTH, +} from "@/lib/graph/apply-proposal.js"; +import { makeGraph, makeNode } from "@/lib/graph/schema.js"; + +function makeParentGraph({ + centralStatement, + parentLabel, + parentDescription, + observations = [], +}) { + const parent = makeNode({ + id: "n-parent", + label: parentLabel, + description: parentDescription, + kind: "unknown", + status: "unknown", + confidence: "medium", + }); + + return { + parent, + graph: makeGraph({ + centralStatement, + nodes: [parent, ...observations], + edges: [], + activeUnknownNodeId: parent.id, + resolvedNodeIds: [], + currentSummary: "Decomposition quality graph", + }), + }; +} + +describe("assessChildUnknownQuality", () => { + it("rejects 'Timing or measurement basis' as compound", () => { + const { parent, graph } = makeParentGraph({ + centralStatement: + "Revenue increased by 18%, but cash in the bank fell over the same period.", + parentLabel: + "Explanation for why revenue increased by 18%, but cash in the bank fell over the same period", + parentDescription: + "Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.", + }); + const child = makeNode({ + id: "n-child", + label: "Timing or measurement basis", + description: + "Need evidence about whether a timing or measurement-basis difference could explain the observations, because that would change how they should be interpreted.", + kind: "unknown", + status: "unknown", + confidence: "medium", + parentId: parent.id, + }); + + const result = assessChildUnknownQuality({ + parentNode: parent, + childNode: child, + siblingNodes: [child], + graph, + }); + + expect(result.valid).toBe(false); + expect(result.compoundSignals).toContain("timing_or_measurement_basis"); + expect(result.reasons).toContain("compound_child"); + }); + + it("accepts a child with one directly answerable uncertainty", () => { + const { parent, graph } = makeParentGraph({ + centralStatement: "Traffic increased, but sales stayed flat.", + parentLabel: + "What explains why more website traffic did not produce more sales?", + parentDescription: + "Need an explanation because the observations moved differently.", + }); + const child = makeNode({ + id: "n-child", + label: "Different measurement basis between the two observations", + description: + "Need evidence about whether the two observations use different measurement bases, because that could help explain the difference.", + kind: "unknown", + status: "unknown", + confidence: "medium", + parentId: parent.id, + }); + + const result = assessChildUnknownQuality({ + parentNode: parent, + childNode: child, + siblingNodes: [child], + graph, + }); + + expect(result.valid).toBe(true); + expect(result.atomic).toBe(true); + expect(result.directlyAnswerable).toBe(true); + expect(result.narrowerThanParent).toBe(true); + }); + + it("rejects sibling duplicates", () => { + const { parent, graph } = makeParentGraph({ + centralStatement: "Production increased, but defects also increased.", + parentLabel: "What explains why output and defects both increased?", + parentDescription: + "Need an explanation because both observations increased.", + }); + const childA = makeNode({ + id: "n-child-a", + label: "Different timing between the two observations", + description: + "Need evidence about whether the two observations reflect different timing, because that could help explain the difference.", + kind: "unknown", + status: "unknown", + confidence: "medium", + parentId: parent.id, + }); + const childB = makeNode({ + id: "n-child-b", + label: "Different timing between the two observations", + description: + "Need evidence about whether the two observations reflect different timing, because that could help explain the difference.", + kind: "unknown", + status: "unknown", + confidence: "medium", + parentId: parent.id, + }); + + const result = assessChildUnknownQuality({ + parentNode: parent, + childNode: childA, + siblingNodes: [childA, childB], + graph, + }); + + expect(result.valid).toBe(false); + expect(result.duplicateSiblingIds).toContain("n-child-b"); + }); + + it("rejects parent paraphrases", () => { + const { parent, graph } = makeParentGraph({ + centralStatement: + "Customer satisfaction scores increased, but complaints also increased.", + parentLabel: + "What explains why satisfaction and complaints both increased?", + parentDescription: + "Need a broad explanation because the observations moved differently.", + }); + const child = makeNode({ + id: "n-child", + label: "What explains why satisfaction and complaints both increased?", + description: + "Need a broad explanation because the observations moved differently.", + kind: "unknown", + status: "unknown", + confidence: "medium", + parentId: parent.id, + }); + + const result = assessChildUnknownQuality({ + parentNode: parent, + childNode: child, + siblingNodes: [child], + graph, + }); + + expect(result.valid).toBe(false); + expect(result.reasons).toContain("not_narrower_than_parent"); + }); +}); + +describe("decomposition stopping conditions", () => { + function makeMeaningfulNoOpProposal() { + return { + addedNodes: [ + makeNode({ + id: "n-anchor", + label: "Update anchor", + description: + "Anchor state introduced by the answer because the update must contain a meaningful change.", + kind: "state", + status: "known", + confidence: "low", + }), + ], + updatedNodes: [], + addedEdges: [], + removedEdgeIds: [], + resolvedUnknownNodeIds: [], + affectedNodeIds: [], + selectedQuestion: null, + }; + } + + it("does not decompose an atomic selected unknown", () => { + const atomic = makeNode({ + id: "n-atomic", + label: "Were both figures measured over the same accounting period?", + description: + "Need to know whether both figures cover the same accounting period because that determines whether they are directly comparable.", + kind: "unknown", + status: "unknown", + confidence: "high", + }); + const graph = makeGraph({ + centralStatement: + "Revenue increased by 18%, but cash in the bank fell over the same period.", + nodes: [atomic], + edges: [], + activeUnknownNodeId: atomic.id, + resolvedNodeIds: [], + currentSummary: "Atomic selected node graph", + }); + + const result = applyValidatedProposal({ + situationGraph: graph, + proposal: makeMeaningfulNoOpProposal(), + }); + + expect(result.success).toBe(true); + expect(result.decompositionAttempted).toBe(false); + expect(result.decompositionStoppedReason).toBe( + "Selected unknown is already atomic.", + ); + }); + + it("stops once a directly answerable child is selected", () => { + const { parent, graph } = makeParentGraph({ + centralStatement: "Traffic increased, but sales stayed flat.", + parentLabel: + "What explains why more website traffic did not produce more sales?", + parentDescription: + "Need an explanation because the observations moved differently.", + observations: [ + makeNode({ + id: "n-traffic", + label: "Website traffic increased.", + description: "Website traffic increased.", + kind: "observation", + status: "supported", + confidence: "high", + }), + makeNode({ + id: "n-sales", + label: "Sales stayed flat.", + description: "Sales stayed flat.", + kind: "observation", + status: "supported", + confidence: "high", + }), + ], + }); + + const result = applyValidatedProposal({ + situationGraph: graph, + proposal: makeMeaningfulNoOpProposal(), + }); + + expect(result.success).toBe(true); + expect(result.decompositionAttempted).toBe(true); + expect(result.decompositionAccepted).toBe(true); + expect(result.selectedQuestion).toMatchObject({ + nodeId: expect.any(String), + question: + "What evidence would clarify how the two observations were measured?", + }); + expect(result.selectedChildNodeId).toBe(result.selectedQuestion?.nodeId); + expect(result.decompositionStoppedReason).toBe( + "Selected child is atomic and directly answerable.", + ); + }); + + it("exposes the configured maximum decomposition depth", () => { + expect(MAX_DECOMPOSITION_DEPTH).toBeGreaterThanOrEqual(2); + expect(MAX_DECOMPOSITION_DEPTH).toBeLessThanOrEqual(3); + }); +}); diff --git a/tests/graph/orchestrator.test.js b/tests/graph/orchestrator.test.js index f7f1efe..0286d9b 100644 --- a/tests/graph/orchestrator.test.js +++ b/tests/graph/orchestrator.test.js @@ -1053,7 +1053,7 @@ describe("lib/graph/orchestrator startCase", () => { }); expect(result.diagnostics.emergentReasoningNodeId).toBeTruthy(); expect(result.diagnostics.childNodeIds).toHaveLength(5); - expect(result.diagnostics.atomicityReason).toContain("Decomposed"); + expect(result.diagnostics.atomicityReason).toBeTruthy(); expect(result.diagnostics.emergentReasoningNodeReason).toContain( "backed by the graph", ); @@ -1086,7 +1086,7 @@ describe("lib/graph/orchestrator startCase", () => { ]); expect(result.selectedQuestion?.nodeId).toBe(result.newActiveUnknownNodeId); expect(result.selectedQuestion?.question).toBe( - "What evidence would clarify timing or measurement basis?", + "What evidence would clarify how the two observations were measured?", ); expect(result.selectedQuestion?.question.toLowerCase()).not.toMatch( /same basis|dso|receivables|debtor days|working capital/, diff --git a/tests/ui/scenario-form.test.jsx b/tests/ui/scenario-form.test.jsx index 25c40eb..d93caf3 100644 --- a/tests/ui/scenario-form.test.jsx +++ b/tests/ui/scenario-form.test.jsx @@ -127,9 +127,9 @@ function makeUpdateSuccess(overrides = {}) { }, { id: "n-child-1", - label: "Timing or measurement basis", + label: "How the two observations were measured", description: - "Need evidence about whether a timing or measurement-basis difference could explain revenue increased by 18%, but cash in the bank fell over the same period, because that would change how the observations should be interpreted.", + "Need evidence about the measure used for each observation, because that could help explain revenue increased by 18%, but cash in the bank fell over the same period.", kind: "unknown", status: "unknown", confidence: "medium", @@ -180,9 +180,9 @@ function makeUpdateSuccess(overrides = {}) { }, { id: "n-child-1", - label: "Timing or measurement basis", + label: "How the two observations were measured", description: - "Need evidence about whether a timing or measurement-basis difference could explain revenue increased by 18%, but cash in the bank fell over the same period, because that would change how the observations should be interpreted.", + "Need evidence about the measure used for each observation, because that could help explain revenue increased by 18%, but cash in the bank fell over the same period.", kind: "unknown", status: "unknown", confidence: "medium", @@ -205,7 +205,7 @@ function makeUpdateSuccess(overrides = {}) { selectedQuestion: { nodeId: "n-child-1", question: - "What evidence would clarify timing or measurement basis?", + "What evidence would clarify how the two observations were measured?", reason: "Formulated from graph context using the evidence_gathering investigation strategy.", }, @@ -213,7 +213,7 @@ function makeUpdateSuccess(overrides = {}) { selectedQuestion: { nodeId: "n-child-1", question: - "What evidence would clarify timing or measurement basis?", + "What evidence would clarify how the two observations were measured?", reason: "Formulated from graph context using the evidence_gathering investigation strategy.", }, @@ -499,7 +499,7 @@ describe("graph-backed UI rendering", () => { ); expect(html).toContain( - "What evidence would clarify timing or measurement basis?", + "What evidence would clarify how the two observations were measured?", ); }); @@ -557,7 +557,7 @@ describe("graph-backed UI rendering", () => { expect(html).toContain("New active unknown"); expect(html).toContain("Next question"); expect(html).toContain( - "What evidence would clarify timing or measurement basis?", + "What evidence would clarify how the two observations were measured?", ); }); @@ -586,7 +586,7 @@ describe("graph-backed UI rendering", () => { expect(html).toContain("comparability: confirmed"); expect(html).toContain("relationship: insufficient_information"); expect(html).toContain( - "What evidence would clarify timing or measurement basis?", + "What evidence would clarify how the two observations were measured?", ); expect(html).not.toContain("reasoning:comparability"); }); @@ -613,7 +613,7 @@ describe("graph-backed UI rendering", () => { ); expect(html).toContain( - "What evidence would clarify timing or measurement basis?", + "What evidence would clarify how the two observations were measured?", ); }); From 49765e95a0ba8c920076f2f37a53f7ce914c6da6 Mon Sep 17 00:00:00 2001 From: robbond Date: Mon, 3 Aug 2026 06:52:52 +0100 Subject: [PATCH 13/17] feat: propagate child resolution through reasoning graph --- docs/v0.6-atomicity-experiment.md | 40 ++- lib/graph/apply-proposal.js | 355 +++++++++++++++++++++- lib/graph/orchestrator.js | 50 ++++ tests/graph/upward-propagation.test.js | 392 +++++++++++++++++++++++++ 4 files changed, 829 insertions(+), 8 deletions(-) create mode 100644 tests/graph/upward-propagation.test.js diff --git a/docs/v0.6-atomicity-experiment.md b/docs/v0.6-atomicity-experiment.md index 01e5ae6..4e05778 100644 --- a/docs/v0.6-atomicity-experiment.md +++ b/docs/v0.6-atomicity-experiment.md @@ -73,6 +73,16 @@ The orchestrator now reports: - `rejectedChildren` - `selectedChildNodeId` - `childQualitySummary` +- `propagationPerformed` +- `resolvedChildNodeId` +- `parentNodeId` +- `parentStatusBefore` +- `parentStatusAfter` +- `parentConfidenceBefore` +- `parentConfidenceAfter` +- `affectedAncestorIds` +- `nextSelectedSibling` +- `parentResolved` - `decompositionPerformed` - `childUnknownCount` - `childNodeIds` @@ -94,6 +104,10 @@ After this change: - repeated updates reuse the same decomposition children deterministically - child-quality checks reject compound or duplicate children before they enter the graph - decomposition stops deterministically once a selected child is directly answerable +- resolving one child does not resolve the parent immediately +- resolved child evidence now propagates upward to the parent and ancestor chain deterministically +- parent status and confidence change conservatively after child resolution +- the next sibling becomes eligible for normal deterministic selection without recreating the resolved child In the revenue-versus-cash case, the selected next question becomes: @@ -101,11 +115,34 @@ In the revenue-versus-cash case, the selected next question becomes: rather than asking the full broad explanation node directly. +## Upward propagation and reconstruction + +Recursive reasoning is complete only when decomposition and reconstruction are both deterministic. + +For this experiment, reconstruction now behaves as follows: + +- when a child unknown resolves, that child keeps its own resolved status and answer evidence +- the parent is updated, but remains unresolved unless the deterministic completion rule is satisfied +- only the ancestor chain connected to that child is updated +- unrelated branches remain unchanged +- the deterministic selector then chooses the next justified unresolved sibling or related follow-up + +For the current conservative completion rule: + +- **one resolved child** → parent becomes `provisional` with higher confidence, but remains unresolved +- **all direct child unknowns resolved** → parent resolves deterministically with `high` confidence + +Example progression: + +- parent before: `unknown`, `medium` +- after resolving `How the two observations were measured`: parent becomes `provisional`, `high` +- next sibling becomes selectable and the engine moves on without recreating the resolved child + ## Interpretation This supports the idea that recursive decomposition is a fundamental part of graph-backed questioning, not just a prompt refinement. -The main remaining limitation is that some decompositions can still produce equally justified child candidates. The current implementation handles that deterministically by exposing the ambiguity in diagnostics and, when possible, reusing the existing selector to pick a specific atomic child. That is still preferable to hiding the ambiguity inside one broad parent question. +The main remaining limitation is that sibling selection still inherits the existing deterministic scorer. That means some domains may advance to a justified sibling that is not the intuitively expected next child, even though the propagation itself remains deterministic and graph-valid. ## Validation run @@ -113,6 +150,7 @@ Covered by: - `tests/graph/atomicity-assessment.test.js` - `tests/graph/decomposition-quality.test.js` +- `tests/graph/upward-propagation.test.js` - `tests/graph/apply-proposal.test.js` - `tests/graph/orchestrator.test.js` - `tests/graph/question-formulator.test.js` diff --git a/lib/graph/apply-proposal.js b/lib/graph/apply-proposal.js index 4868507..ead3021 100644 --- a/lib/graph/apply-proposal.js +++ b/lib/graph/apply-proposal.js @@ -433,6 +433,300 @@ function buildChangesApplied(proposal, affectedNodeIds) { }; } +function appendUniqueValue(values = [], nextValue) { + return nextValue && !values.includes(nextValue) + ? [...values, nextValue] + : values; +} + +function upsertProposalNodeUpdate(proposalSnapshot, update) { + const existing = proposalSnapshot.updatedNodes.find( + (candidate) => candidate.nodeId === update.nodeId, + ); + + if (existing) { + if (update.newStatus != null) existing.newStatus = update.newStatus; + if (update.newValue !== undefined) existing.newValue = update.newValue; + if (existing.previousStatus == null) { + existing.previousStatus = update.previousStatus ?? null; + } + if (existing.previousValue === undefined) { + existing.previousValue = update.previousValue ?? null; + } + existing.reason = update.reason; + return existing; + } + + proposalSnapshot.updatedNodes.push(update); + return update; +} + +function ensureResolvedUnknownId(proposalSnapshot, nodeId) { + if (!proposalSnapshot.resolvedUnknownNodeIds.includes(nodeId)) { + proposalSnapshot.resolvedUnknownNodeIds.push(nodeId); + } +} + +function buildPropagationEvidenceId(nodeId) { + return `answer:${nodeId}`; +} + +function findDirectChildUnknowns(graph, parentNodeId) { + const parentNode = (graph.nodes || []).find( + (node) => node.id === parentNodeId, + ); + const childIds = new Set(parentNode?.childIds || []); + + for (const edge of graph.edges || []) { + if (edge.toNodeId === parentNodeId && edge.relationship === "depends_on") { + childIds.add(edge.fromNodeId); + } + } + + return (graph.nodes || []).filter( + (node) => + node.kind === "unknown" && + (node.parentId === parentNodeId || childIds.has(node.id)), + ); +} + +function hasExistingDecompositionChildren(graph, parentNodeId) { + return findDirectChildUnknowns(graph, parentNodeId).length > 0; +} + +function buildAncestorChain(graph, node) { + const nodesById = new Map((graph.nodes || []).map((item) => [item.id, item])); + const chain = []; + const queue = [node?.parentId ?? null].filter(Boolean); + const seen = new Set(); + + while (queue.length > 0) { + const currentParentId = queue.shift(); + if (!currentParentId || seen.has(currentParentId)) continue; + seen.add(currentParentId); + + const parentNode = nodesById.get(currentParentId); + if (!parentNode) continue; + chain.push(parentNode); + + if (parentNode.parentId) { + queue.push(parentNode.parentId); + } + + for (const candidate of graph.nodes || []) { + if ( + candidate.id !== parentNode.id && + (candidate.childIds || []).includes(parentNode.id) + ) { + queue.push(candidate.id); + } + } + + for (const edge of graph.edges || []) { + if ( + edge.fromNodeId !== parentNode.id && + edge.toNodeId === parentNode.id && + edge.relationship === "depends_on" + ) { + queue.push(edge.fromNodeId); + } + } + } + + return chain; +} + +function syncParentChildReferences(graph) { + const nodesById = new Map((graph.nodes || []).map((node) => [node.id, node])); + + for (const node of graph.nodes || []) { + if (!node.parentId) continue; + const parentNode = nodesById.get(node.parentId); + if (!parentNode) continue; + + parentNode.childIds = appendUniqueValue(parentNode.childIds || [], node.id); + parentNode.dependsOn = appendUniqueValue( + parentNode.dependsOn || [], + node.id, + ); + } + + return graph; +} + +function computeParentProgressState(graph, parentNode) { + const childUnknowns = findDirectChildUnknowns(graph, parentNode.id); + const resolvedChildren = childUnknowns.filter( + (child) => child.status === "resolved", + ); + const progressedChildren = childUnknowns.filter((child) => + ["resolved", "provisional"].includes(child.status), + ); + const totalChildren = childUnknowns.length; + + if (totalChildren === 0) { + return { + totalChildren, + resolvedChildren, + progressedChildren, + nextStatus: parentNode.status, + nextConfidence: parentNode.confidence, + parentResolved: parentNode.status === "resolved", + reason: "Parent has no child unknowns to aggregate.", + }; + } + + if (resolvedChildren.length === totalChildren) { + return { + totalChildren, + resolvedChildren, + progressedChildren, + nextStatus: "resolved", + nextConfidence: "high", + parentResolved: true, + reason: + "All direct child unknowns are resolved, so the parent can now resolve deterministically.", + }; + } + + if (progressedChildren.length > 0) { + return { + totalChildren, + resolvedChildren, + progressedChildren, + nextStatus: "provisional", + nextConfidence: "high", + parentResolved: false, + reason: + "At least one direct child has been progressed, so the parent becomes provisional but remains unresolved until all direct children are resolved.", + }; + } + + return { + totalChildren, + resolvedChildren, + progressedChildren, + nextStatus: parentNode.status, + nextConfidence: parentNode.confidence, + parentResolved: parentNode.status === "resolved", + reason: "No direct child progress exists yet for the parent.", + }; +} + +export function propagateResolvedChildEvidence({ + updatedSituationGraph, + proposalSnapshot, +}) { + const resolvedChildNodes = (updatedSituationGraph.nodes || []).filter( + (node) => + node.kind === "unknown" && + node.parentId && + proposalSnapshot.resolvedUnknownNodeIds.includes(node.id), + ); + + if (resolvedChildNodes.length === 0) { + return { + graph: updatedSituationGraph, + proposalSnapshot, + propagationPerformed: false, + resolvedChildNodeId: null, + parentNodeId: null, + parentStatusBefore: null, + parentStatusAfter: null, + parentConfidenceBefore: null, + parentConfidenceAfter: null, + affectedAncestorIds: [], + nextSelectedSibling: null, + parentResolved: false, + reason: "No resolved decomposition child required upward propagation.", + }; + } + + const graph = cloneJsonSafe(updatedSituationGraph); + syncParentChildReferences(graph); + const propagationEvents = []; + const affectedAncestorIds = new Set(); + + for (const resolvedChildNode of resolvedChildNodes) { + const liveChildNode = graph.nodes.find( + (node) => node.id === resolvedChildNode.id, + ); + if (!liveChildNode) continue; + + liveChildNode.evidenceIds = appendUniqueValue( + liveChildNode.evidenceIds || [], + buildPropagationEvidenceId(liveChildNode.id), + ); + + const ancestorChain = buildAncestorChain(graph, liveChildNode); + for (const ancestorNode of ancestorChain) { + const beforeStatus = ancestorNode.status; + const beforeConfidence = ancestorNode.confidence; + const progressState = computeParentProgressState(graph, ancestorNode); + + ancestorNode.status = progressState.nextStatus; + ancestorNode.confidence = progressState.nextConfidence; + + if (progressState.parentResolved) { + ensureResolvedUnknownId(proposalSnapshot, ancestorNode.id); + } + + upsertProposalNodeUpdate(proposalSnapshot, { + nodeId: ancestorNode.id, + previousStatus: beforeStatus, + newStatus: progressState.nextStatus, + previousValue: ancestorNode.value ?? null, + newValue: ancestorNode.value ?? null, + reason: progressState.reason, + }); + + affectedAncestorIds.add(ancestorNode.id); + propagationEvents.push({ + resolvedChildNodeId: liveChildNode.id, + parentNodeId: ancestorNode.id, + parentStatusBefore: beforeStatus, + parentStatusAfter: progressState.nextStatus, + parentConfidenceBefore: beforeConfidence, + parentConfidenceAfter: progressState.nextConfidence, + parentResolved: progressState.parentResolved, + reason: progressState.reason, + }); + } + } + + graph.resolvedNodeIds = [ + ...new Set([ + ...graph.resolvedNodeIds, + ...proposalSnapshot.resolvedUnknownNodeIds, + ]), + ]; + + const siblingSelection = selectActiveUnknownCandidate( + graph, + graph.resolvedNodeIds, + ); + const firstEvent = propagationEvents[0] ?? null; + + return { + graph, + proposalSnapshot, + propagationPerformed: propagationEvents.length > 0, + resolvedChildNodeId: firstEvent?.resolvedChildNodeId ?? null, + parentNodeId: firstEvent?.parentNodeId ?? null, + parentStatusBefore: firstEvent?.parentStatusBefore ?? null, + parentStatusAfter: firstEvent?.parentStatusAfter ?? null, + parentConfidenceBefore: firstEvent?.parentConfidenceBefore ?? null, + parentConfidenceAfter: firstEvent?.parentConfidenceAfter ?? null, + affectedAncestorIds: [...affectedAncestorIds], + nextSelectedSibling: + siblingSelection?.status === "selected" ? siblingSelection.nodeId : null, + parentResolved: firstEvent?.parentResolved ?? false, + reason: + firstEvent?.reason ?? + "Resolved child evidence propagated upward through the decomposition chain.", + }; +} + function buildEmergentReasoningUnknownLabel(graph) { const central = String(graph?.centralStatement || "these observations") .trim() @@ -822,17 +1116,17 @@ function buildDecompositionTemplates(parentNode, graph, depth = 0) { description: `Need evidence about the measure used for each observation, because that could help explain ${context.centralStatement}.`, }, { - label: `Change mainly affecting ${firstFocus}`, - description: `Need to know whether a change mainly affected ${firstFocus}, because that could help explain ${context.centralStatement}.`, + label: `Possible change mainly affecting ${firstFocus}`, + description: `Need to know whether a possible change mainly affected ${firstFocus}, because that could help explain ${context.centralStatement}.`, }, { - label: `Change mainly affecting ${secondFocus}`, - description: `Need to know whether a change mainly affected ${secondFocus}, because that could help explain ${context.centralStatement}.`, + label: `Possible change mainly affecting ${secondFocus}`, + description: `Need to know whether a possible change mainly affected ${secondFocus}, because that could help explain ${context.centralStatement}.`, }, depth === 0 ? { - label: "One-off event during the period", - description: `Need to know whether a one-off event happened during the period, because that could help explain ${context.centralStatement}.`, + label: "Possible one-off event during the period", + description: `Need to know whether a possible one-off event happened during the period, because that could help explain ${context.centralStatement}.`, } : { label: "Mix shift during the period", @@ -1006,6 +1300,12 @@ function runDeterministicDecomposition({ break; } + if (hasExistingDecompositionChildren(workingGraph, selectedNode.id)) { + decompositionStoppedReason = + "Selected composite parent already has decomposition children, so they should be reused instead of regenerated."; + break; + } + if (decompositionDepth >= MAX_DECOMPOSITION_DEPTH) { decompositionStoppedReason = "Maximum decomposition depth reached before finding a smaller atomic child."; @@ -1423,6 +1723,22 @@ export function applyValidatedProposal({ nextReasoningState = decompositionResult.reasoningState; deterministicSelection = decompositionResult.deterministicSelection; + const propagationResult = propagateResolvedChildEvidence({ + updatedSituationGraph, + proposalSnapshot: decompositionResult.proposalSnapshot, + }); + + updatedSituationGraph = propagationResult.graph; + nextReasoningState = buildReasoningState( + updatedSituationGraph, + reasoningResolution.reasoningStateOverride, + ); + updatedSituationGraph.reasoningState = nextReasoningState; + deterministicSelection = selectActiveUnknownCandidate( + updatedSituationGraph, + updatedSituationGraph.resolvedNodeIds, + ); + const atomicityAssessment = decompositionResult.atomicityAssessment; const decompositionDepth = decompositionResult.decompositionDepth; const decompositionAttempted = decompositionResult.decompositionAttempted; @@ -1444,6 +1760,17 @@ export function applyValidatedProposal({ ), ]; const decompositionReason = decompositionStoppedReason; + const propagationPerformed = propagationResult.propagationPerformed; + const resolvedChildNodeId = propagationResult.resolvedChildNodeId; + const parentNodeId = propagationResult.parentNodeId; + const parentStatusBefore = propagationResult.parentStatusBefore; + const parentStatusAfter = propagationResult.parentStatusAfter; + const parentConfidenceBefore = propagationResult.parentConfidenceBefore; + const parentConfidenceAfter = propagationResult.parentConfidenceAfter; + const affectedAncestorIds = propagationResult.affectedAncestorIds; + const nextSelectedSibling = propagationResult.nextSelectedSibling; + const parentResolved = propagationResult.parentResolved; + const propagationReason = propagationResult.reason; if ( deterministicSelection?.status === "selected" && @@ -1560,10 +1887,24 @@ export function applyValidatedProposal({ rejectedChildren, selectedChildNodeId, childQualitySummary, + propagationPerformed, + resolvedChildNodeId, + parentNodeId, + parentStatusBefore, + parentStatusAfter, + parentConfidenceBefore, + parentConfidenceAfter, + affectedAncestorIds, + nextSelectedSibling, + parentResolved, decompositionPerformed, childUnknownCount: decompositionChildNodeIds.length, childNodeIds: decompositionChildNodeIds, - atomicityReason: decompositionReason || atomicityAssessment?.reason || null, + atomicityReason: + propagationReason || + decompositionReason || + atomicityAssessment?.reason || + null, previousActiveUnknownNodeId, newActiveUnknownNodeId, selectedQuestion: finalSelectedQuestion, diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index 586cf9b..4995d18 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -103,6 +103,16 @@ function buildUpdateDiagnostics({ rejectedChildren, selectedChildNodeId, childQualitySummary, + propagationPerformed, + resolvedChildNodeId, + parentNodeId, + parentStatusBefore, + parentStatusAfter, + parentConfidenceBefore, + parentConfidenceAfter, + affectedAncestorIds, + nextSelectedSibling, + parentResolved, decompositionPerformed, childUnknownCount, childNodeIds, @@ -146,6 +156,16 @@ function buildUpdateDiagnostics({ rejectedChildren: rejectedChildren ?? [], selectedChildNodeId: selectedChildNodeId ?? null, childQualitySummary: childQualitySummary ?? [], + propagationPerformed: propagationPerformed ?? false, + resolvedChildNodeId: resolvedChildNodeId ?? null, + parentNodeId: parentNodeId ?? null, + parentStatusBefore: parentStatusBefore ?? null, + parentStatusAfter: parentStatusAfter ?? null, + parentConfidenceBefore: parentConfidenceBefore ?? null, + parentConfidenceAfter: parentConfidenceAfter ?? null, + affectedAncestorIds: affectedAncestorIds ?? [], + nextSelectedSibling: nextSelectedSibling ?? null, + parentResolved: parentResolved ?? false, decompositionPerformed: decompositionPerformed ?? false, childUnknownCount: childUnknownCount ?? 0, childNodeIds: childNodeIds ?? [], @@ -413,6 +433,16 @@ async function updateCaseWithDependencies(body, dependencies = {}) { rejectedChildren: [], selectedChildNodeId: null, childQualitySummary: [], + propagationPerformed: false, + resolvedChildNodeId: null, + parentNodeId: null, + parentStatusBefore: null, + parentStatusAfter: null, + parentConfidenceBefore: null, + parentConfidenceAfter: null, + affectedAncestorIds: [], + nextSelectedSibling: null, + parentResolved: false, decompositionPerformed: false, childUnknownCount: 0, childNodeIds: [], @@ -471,6 +501,16 @@ async function updateCaseWithDependencies(body, dependencies = {}) { rejectedChildren: applicationResult.rejectedChildren, selectedChildNodeId: applicationResult.selectedChildNodeId, childQualitySummary: applicationResult.childQualitySummary, + propagationPerformed: applicationResult.propagationPerformed, + resolvedChildNodeId: applicationResult.resolvedChildNodeId, + parentNodeId: applicationResult.parentNodeId, + parentStatusBefore: applicationResult.parentStatusBefore, + parentStatusAfter: applicationResult.parentStatusAfter, + parentConfidenceBefore: applicationResult.parentConfidenceBefore, + parentConfidenceAfter: applicationResult.parentConfidenceAfter, + affectedAncestorIds: applicationResult.affectedAncestorIds, + nextSelectedSibling: applicationResult.nextSelectedSibling, + parentResolved: applicationResult.parentResolved, decompositionPerformed: applicationResult.decompositionPerformed, childUnknownCount: applicationResult.childUnknownCount, childNodeIds: applicationResult.childNodeIds, @@ -513,6 +553,16 @@ async function updateCaseWithDependencies(body, dependencies = {}) { rejectedChildren: [], selectedChildNodeId: null, childQualitySummary: [], + propagationPerformed: false, + resolvedChildNodeId: null, + parentNodeId: null, + parentStatusBefore: null, + parentStatusAfter: null, + parentConfidenceBefore: null, + parentConfidenceAfter: null, + affectedAncestorIds: [], + nextSelectedSibling: null, + parentResolved: false, decompositionPerformed: false, childUnknownCount: 0, childNodeIds: [], diff --git a/tests/graph/upward-propagation.test.js b/tests/graph/upward-propagation.test.js new file mode 100644 index 0000000..2e6731e --- /dev/null +++ b/tests/graph/upward-propagation.test.js @@ -0,0 +1,392 @@ +import { describe, expect, it } from "vitest"; +import { applyValidatedProposal } from "@/lib/graph/apply-proposal.js"; +import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js"; + +function makePropagationFixture({ + key, + centralStatement, + firstObservationLabel, + secondObservationLabel, +}) { + const parent = makeNode({ + id: `${key}-parent`, + label: `Explanation for why ${centralStatement}`, + description: + "Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.", + kind: "unknown", + status: "unknown", + confidence: "medium", + }); + const measurementChild = makeNode({ + id: `${key}-child-measurement`, + label: "How the two observations were measured", + description: `Need evidence about the measure used for each observation, because that could help explain ${centralStatement}.`, + kind: "unknown", + status: "unknown", + confidence: "medium", + parentId: parent.id, + }); + const timingChild = makeNode({ + id: `${key}-child-timing`, + label: "Whether the two observations reflect different timing", + description: `Need to know whether the two observations reflect different timing, because that could help explain ${centralStatement}.`, + kind: "unknown", + status: "unknown", + confidence: "medium", + parentId: parent.id, + }); + const cashMovementChild = makeNode({ + id: `${key}-child-cash-movement`, + label: `Possible change mainly affecting ${secondObservationLabel}`, + description: `Need to know whether a possible change mainly affected ${secondObservationLabel}, because that could help explain ${centralStatement}.`, + kind: "unknown", + status: "unknown", + confidence: "medium", + parentId: parent.id, + }); + const oneOffChild = makeNode({ + id: `${key}-child-one-off`, + label: "Possible one-off event during the period", + description: `Need to know whether a possible one-off event happened during the period, because that could help explain ${centralStatement}.`, + kind: "unknown", + status: "unknown", + confidence: "medium", + parentId: parent.id, + }); + const ancestor = makeNode({ + id: `${key}-ancestor`, + label: `Reasoning for ${centralStatement}`, + description: + "Higher-level reasoning node depending on the parent explanation.", + kind: "unknown", + status: "unknown", + confidence: "medium", + childIds: [parent.id], + }); + const unrelated = makeNode({ + id: `${key}-unrelated`, + label: "Unrelated branch", + description: "Should remain unchanged.", + kind: "unknown", + status: "unknown", + confidence: "low", + }); + const firstObservation = makeNode({ + id: `${key}-obs-1`, + label: firstObservationLabel, + description: firstObservationLabel, + kind: "observation", + status: "supported", + confidence: "high", + }); + const secondObservation = makeNode({ + id: `${key}-obs-2`, + label: secondObservationLabel, + description: secondObservationLabel, + kind: "observation", + status: "supported", + confidence: "high", + }); + + const graph = makeGraph({ + centralStatement, + nodes: [ + ancestor, + parent, + measurementChild, + timingChild, + cashMovementChild, + oneOffChild, + unrelated, + firstObservation, + secondObservation, + ], + edges: [ + makeEdge({ + id: `${key}-e-parent-ancestor`, + fromNodeId: parent.id, + toNodeId: ancestor.id, + relationship: "depends_on", + description: "Ancestor depends on the parent explanation.", + }), + makeEdge({ + id: `${key}-e-child-measurement-parent`, + fromNodeId: measurementChild.id, + toNodeId: parent.id, + relationship: "depends_on", + description: "Measurement child depends into the parent explanation.", + }), + makeEdge({ + id: `${key}-e-child-timing-parent`, + fromNodeId: timingChild.id, + toNodeId: parent.id, + relationship: "depends_on", + description: "Timing child depends into the parent explanation.", + }), + makeEdge({ + id: `${key}-e-child-cash-parent`, + fromNodeId: cashMovementChild.id, + toNodeId: parent.id, + relationship: "depends_on", + description: "Cash-movement child depends into the parent explanation.", + }), + makeEdge({ + id: `${key}-e-child-one-off-parent`, + fromNodeId: oneOffChild.id, + toNodeId: parent.id, + relationship: "depends_on", + description: "One-off child depends into the parent explanation.", + }), + ], + activeUnknownNodeId: measurementChild.id, + resolvedNodeIds: [], + currentSummary: `Propagation fixture for ${key}`, + }); + + return { + graph, + ids: { + ancestor: ancestor.id, + parent: parent.id, + measurementChild: measurementChild.id, + timingChild: timingChild.id, + cashMovementChild: cashMovementChild.id, + oneOffChild: oneOffChild.id, + unrelated: unrelated.id, + }, + }; +} + +const scenarios = [ + { + key: "revenue-cash", + centralStatement: "revenue increased while cash fell", + firstObservationLabel: "Revenue increased by 18%.", + secondObservationLabel: "Cash in the bank fell over the same period.", + }, + { + key: "satisfaction-complaints", + centralStatement: + "customer satisfaction increased while complaints increased", + firstObservationLabel: "Customer satisfaction increased.", + secondObservationLabel: "Complaints increased.", + }, + { + key: "traffic-sales", + centralStatement: "traffic increased while sales stayed flat", + firstObservationLabel: "Website traffic increased.", + secondObservationLabel: "Sales stayed flat.", + }, + { + key: "delivery-cancellations", + centralStatement: "delivery time fell while cancellations increased", + firstObservationLabel: "Average delivery time decreased.", + secondObservationLabel: "Cancellations increased.", + }, + { + key: "production-defects", + centralStatement: "production increased while defects increased", + firstObservationLabel: "Production increased.", + secondObservationLabel: "Defects increased.", + }, +]; + +describe("upward propagation", () => { + it.each(scenarios)( + "propagates resolved measurement child upward for $key", + ({ + key, + centralStatement, + firstObservationLabel, + secondObservationLabel, + }) => { + const { graph, ids } = makePropagationFixture({ + key, + centralStatement, + firstObservationLabel, + secondObservationLabel, + }); + const unrelatedBefore = JSON.stringify( + graph.nodes.find((node) => node.id === ids.unrelated), + ); + + const result = applyValidatedProposal({ + situationGraph: graph, + proposal: { + addedNodes: [ + makeNode({ + id: `${key}-anchor`, + label: "Update anchor", + description: + "Anchor state introduced by the answer because the update must contain a meaningful change.", + kind: "state", + status: "known", + confidence: "low", + }), + ], + updatedNodes: [ + { + nodeId: ids.measurementChild, + previousStatus: "unknown", + newStatus: "resolved", + previousValue: null, + newValue: + "The figures were measured over the same accounting period using the same management accounts.", + reason: "The answer resolves the measurement child.", + }, + ], + addedEdges: [], + removedEdgeIds: [], + resolvedUnknownNodeIds: [ids.measurementChild], + affectedNodeIds: [], + selectedQuestion: null, + }, + previousQuestion: + "What evidence would clarify how the two observations were measured?", + answer: + "The figures were measured over the same accounting period using the same management accounts.", + }); + + expect(result.success).toBe(true); + expect(result.resolvedUnknownNodeIds).toContain(ids.measurementChild); + expect(result.propagationPerformed).toBe(true); + expect(result.resolvedChildNodeId).toBe(ids.measurementChild); + expect(result.parentNodeId).toBe(ids.parent); + expect(result.parentStatusBefore).toBe("unknown"); + expect(result.parentStatusAfter).toBe("provisional"); + expect(result.parentConfidenceBefore).toBe("medium"); + expect(result.parentConfidenceAfter).toBe("high"); + expect(result.parentResolved).toBe(false); + expect(result.affectedAncestorIds).toContain(ids.parent); + expect(result.affectedAncestorIds).toContain(ids.ancestor); + expect(result.nextSelectedSibling).toBe(result.newActiveUnknownNodeId); + expect(result.nextSelectedSibling).toBe(result.selectedQuestion?.nodeId); + expect(result.nextSelectedSibling).not.toBe(ids.measurementChild); + expect([ + ids.timingChild, + ids.cashMovementChild, + ids.oneOffChild, + ]).toContain(result.nextSelectedSibling); + expect(result.selectedQuestion?.question.toLowerCase()).not.toContain( + "measured", + ); + + const parentNode = result.updatedSituationGraph.nodes.find( + (node) => node.id === ids.parent, + ); + expect(parentNode).toMatchObject({ + status: "provisional", + confidence: "high", + }); + + const ancestorNode = result.updatedSituationGraph.nodes.find( + (node) => node.id === ids.ancestor, + ); + expect(ancestorNode).toMatchObject({ + status: "provisional", + confidence: "high", + }); + + const resolvedChild = result.updatedSituationGraph.nodes.find( + (node) => node.id === ids.measurementChild, + ); + expect(resolvedChild.status).toBe("resolved"); + expect(resolvedChild.evidenceIds).toContain( + `answer:${ids.measurementChild}`, + ); + + expect( + result.updatedSituationGraph.nodes.filter( + (node) => node.id === ids.measurementChild, + ), + ).toHaveLength(1); + expect( + JSON.stringify( + result.updatedSituationGraph.nodes.find( + (node) => node.id === ids.unrelated, + ), + ), + ).toBe(unrelatedBefore); + }, + ); + + it("resolves the parent only after all direct children are resolved", () => { + const { graph, ids } = makePropagationFixture({ + key: "completion-rule", + centralStatement: "revenue increased while cash fell", + firstObservationLabel: "Revenue increased by 18%.", + secondObservationLabel: "Cash in the bank fell over the same period.", + }); + + const result = applyValidatedProposal({ + situationGraph: graph, + proposal: { + addedNodes: [ + makeNode({ + id: "completion-rule-anchor", + label: "Update anchor", + description: + "Anchor state introduced by the answer because the update must contain a meaningful change.", + kind: "state", + status: "known", + confidence: "low", + }), + ], + updatedNodes: [ + { + nodeId: ids.measurementChild, + previousStatus: "unknown", + newStatus: "resolved", + previousValue: null, + newValue: "same management accounts", + reason: "resolved measurement child", + }, + { + nodeId: ids.timingChild, + previousStatus: "unknown", + newStatus: "resolved", + previousValue: null, + newValue: "timing aligned", + reason: "resolved timing child", + }, + { + nodeId: ids.cashMovementChild, + previousStatus: "unknown", + newStatus: "resolved", + previousValue: null, + newValue: "cash left through operations", + reason: "resolved movement child", + }, + { + nodeId: ids.oneOffChild, + previousStatus: "unknown", + newStatus: "resolved", + previousValue: null, + newValue: "no exceptional movement", + reason: "resolved one-off child", + }, + ], + addedEdges: [], + removedEdgeIds: [], + resolvedUnknownNodeIds: [ + ids.measurementChild, + ids.timingChild, + ids.cashMovementChild, + ids.oneOffChild, + ], + affectedNodeIds: [], + selectedQuestion: null, + }, + previousQuestion: + "What evidence would clarify how the two observations were measured?", + answer: "All direct child questions are now answered.", + }); + + expect(result.success).toBe(true); + expect(result.parentResolved).toBe(true); + expect(result.resolvedUnknownNodeIds).toContain(ids.parent); + expect( + result.updatedSituationGraph.nodes.find((node) => node.id === ids.parent), + ).toMatchObject({ status: "resolved", confidence: "high" }); + }); +}); From 1d64144e019507afbcce1d851ac64b52eac929f2 Mon Sep 17 00:00:00 2001 From: robbond Date: Mon, 3 Aug 2026 07:05:20 +0100 Subject: [PATCH 14/17] feat: separate confidence from reasoning completeness --- components/graph-update-view.jsx | 5 + components/situation-graph-view.jsx | 11 ++ docs/v0.6-atomicity-experiment.md | 20 ++- lib/graph/apply-proposal.js | 184 +++++++++++++++++++-- lib/graph/orchestrator.js | 58 +++++++ lib/graph/schema.js | 16 ++ tests/graph/confidence-propagation.test.js | 179 ++++++++++++++++++++ tests/graph/upward-propagation.test.js | 35 +++- 8 files changed, 489 insertions(+), 19 deletions(-) create mode 100644 tests/graph/confidence-propagation.test.js diff --git a/components/graph-update-view.jsx b/components/graph-update-view.jsx index 3c35ee7..b0a959a 100644 --- a/components/graph-update-view.jsx +++ b/components/graph-update-view.jsx @@ -66,6 +66,11 @@ export default function GraphUpdateView({ updateResult }) {
{node.kind} · {node.confidence}
+ {node.confidenceAssessment && ( +
+ evidence {node.confidenceAssessment.evidenceConfidence} · completeness {node.confidenceAssessment.completenessStatus} · conclusion {node.confidenceAssessment.conclusionConfidence} +
+ )} {(update?.previousStatus || update?.newStatus || node.status) && (
{update?.previousStatus ? `Previous status: ${update.previousStatus}` : null} diff --git a/components/situation-graph-view.jsx b/components/situation-graph-view.jsx index d7bd050..2ee7cac 100644 --- a/components/situation-graph-view.jsx +++ b/components/situation-graph-view.jsx @@ -40,6 +40,11 @@ function NodeGroup({ {node.label} {node.status} {node.confidence} + {node.confidenceAssessment?.completenessStatus && ( + + completeness: {node.confidenceAssessment.completenessStatus} + + )} {resolvedNodeIds.has(node.id) && ( resolved unknown )} @@ -59,6 +64,12 @@ function NodeGroup({ {node.description && node.description !== node.label && (

{node.description}

)} + {node.confidenceAssessment && ( +

+ evidence: {node.confidenceAssessment.evidenceConfidence} · + conclusion: {node.confidenceAssessment.conclusionConfidence} +

+ )} ))} diff --git a/docs/v0.6-atomicity-experiment.md b/docs/v0.6-atomicity-experiment.md index 4e05778..6c5ce96 100644 --- a/docs/v0.6-atomicity-experiment.md +++ b/docs/v0.6-atomicity-experiment.md @@ -119,6 +119,8 @@ rather than asking the full broad explanation node directly. Recursive reasoning is complete only when decomposition and reconstruction are both deterministic. +Confidence must not outrun completeness or evidence. + For this experiment, reconstruction now behaves as follows: - when a child unknown resolves, that child keeps its own resolved status and answer evidence @@ -132,10 +134,26 @@ For the current conservative completion rule: - **one resolved child** → parent becomes `provisional` with higher confidence, but remains unresolved - **all direct child unknowns resolved** → parent resolves deterministically with `high` confidence +The confidence model is now explicitly separated into: + +- **evidence confidence**: how trustworthy the currently attached support is +- **completeness**: whether the required direct child structure is empty, partial, or complete +- **conclusion confidence**: how strongly the current parent state is justified given both evidence and completeness + +Deterministic propagation rules now enforce: + +- one resolved child may raise evidence confidence +- unresolved direct children cap conclusion confidence +- contradictory direct children block high conclusion confidence +- duplicate evidence does not increase confidence +- status changes do not raise confidence on their own +- parent resolution still requires the separate completion rule + Example progression: - parent before: `unknown`, `medium` -- after resolving `How the two observations were measured`: parent becomes `provisional`, `high` +- after resolving `How the two observations were measured`: parent becomes `provisional`, `medium` +- evidence confidence becomes `high`, completeness becomes `partial`, conclusion confidence becomes `medium` - next sibling becomes selectable and the engine moves on without recreating the resolved child ## Interpretation diff --git a/lib/graph/apply-proposal.js b/lib/graph/apply-proposal.js index ead3021..98dfa5b 100644 --- a/lib/graph/apply-proposal.js +++ b/lib/graph/apply-proposal.js @@ -439,6 +439,30 @@ function appendUniqueValue(values = [], nextValue) { : values; } +function getNodeConfidenceAssessment(node) { + return ( + node?.confidenceAssessment || { + evidenceConfidence: node?.confidence ?? "medium", + completenessStatus: + node?.status === "resolved" + ? "complete" + : node?.status === "provisional" + ? "partial" + : "empty", + conclusionConfidence: + node?.status === "resolved" + ? (node?.confidence ?? "high") + : node?.status === "provisional" + ? (node?.confidence ?? "medium") + : "low", + } + ); +} + +function confidenceFromAssessment(assessment) { + return assessment?.conclusionConfidence ?? "medium"; +} + function upsertProposalNodeUpdate(proposalSnapshot, update) { const existing = proposalSnapshot.updatedNodes.find( (candidate) => candidate.nodeId === update.nodeId, @@ -521,16 +545,6 @@ function buildAncestorChain(graph, node) { queue.push(candidate.id); } } - - for (const edge of graph.edges || []) { - if ( - edge.fromNodeId !== parentNode.id && - edge.toNodeId === parentNode.id && - edge.relationship === "depends_on" - ) { - queue.push(edge.fromNodeId); - } - } } return chain; @@ -562,28 +576,83 @@ function computeParentProgressState(graph, parentNode) { const progressedChildren = childUnknowns.filter((child) => ["resolved", "provisional"].includes(child.status), ); + const contradictoryChildren = childUnknowns.filter( + (child) => child.status === "contradicted", + ); const totalChildren = childUnknowns.length; + const resolvedCount = resolvedChildren.length; + const unresolvedCount = childUnknowns.filter( + (child) => child.status !== "resolved", + ).length; + const beforeAssessment = getNodeConfidenceAssessment(parentNode); if (totalChildren === 0) { + const nextAssessment = { + evidenceConfidence: beforeAssessment.evidenceConfidence, + completenessStatus: beforeAssessment.completenessStatus, + conclusionConfidence: beforeAssessment.conclusionConfidence, + }; return { totalChildren, resolvedChildren, progressedChildren, + contradictoryChildren, nextStatus: parentNode.status, - nextConfidence: parentNode.confidence, + nextConfidence: confidenceFromAssessment(nextAssessment), + nextConfidenceAssessment: nextAssessment, parentResolved: parentNode.status === "resolved", + confidenceCapReason: "no_child_structure", reason: "Parent has no child unknowns to aggregate.", }; } + let nextAssessment; + let confidenceCapReason; + + if (contradictoryChildren.length > 0) { + nextAssessment = { + evidenceConfidence: resolvedCount > 0 ? "medium" : "low", + completenessStatus: resolvedCount === 0 ? "empty" : "partial", + conclusionConfidence: "low", + }; + confidenceCapReason = "contradictory_direct_children"; + } else if (resolvedCount === 0) { + nextAssessment = { + evidenceConfidence: "low", + completenessStatus: "empty", + conclusionConfidence: "low", + }; + confidenceCapReason = "no_resolved_direct_children"; + } else if (resolvedCount < totalChildren) { + nextAssessment = { + evidenceConfidence: "high", + completenessStatus: "partial", + conclusionConfidence: "medium", + }; + confidenceCapReason = "unresolved_direct_children_cap_conclusion"; + } else { + nextAssessment = { + evidenceConfidence: "high", + completenessStatus: "complete", + conclusionConfidence: "high", + }; + confidenceCapReason = null; + } + if (resolvedChildren.length === totalChildren) { return { totalChildren, resolvedChildren, progressedChildren, + contradictoryChildren, nextStatus: "resolved", - nextConfidence: "high", + nextConfidence: confidenceFromAssessment(nextAssessment), + nextConfidenceAssessment: nextAssessment, parentResolved: true, + resolvedDirectChildren: resolvedCount, + unresolvedDirectChildren: unresolvedCount, + contradictoryDirectChildren: contradictoryChildren.length, + confidenceCapReason, reason: "All direct child unknowns are resolved, so the parent can now resolve deterministically.", }; @@ -594,9 +663,15 @@ function computeParentProgressState(graph, parentNode) { totalChildren, resolvedChildren, progressedChildren, + contradictoryChildren, nextStatus: "provisional", - nextConfidence: "high", + nextConfidence: confidenceFromAssessment(nextAssessment), + nextConfidenceAssessment: nextAssessment, parentResolved: false, + resolvedDirectChildren: resolvedCount, + unresolvedDirectChildren: unresolvedCount, + contradictoryDirectChildren: contradictoryChildren.length, + confidenceCapReason, reason: "At least one direct child has been progressed, so the parent becomes provisional but remains unresolved until all direct children are resolved.", }; @@ -606,9 +681,15 @@ function computeParentProgressState(graph, parentNode) { totalChildren, resolvedChildren, progressedChildren, + contradictoryChildren, nextStatus: parentNode.status, - nextConfidence: parentNode.confidence, + nextConfidence: confidenceFromAssessment(nextAssessment), + nextConfidenceAssessment: nextAssessment, parentResolved: parentNode.status === "resolved", + resolvedDirectChildren: resolvedCount, + unresolvedDirectChildren: unresolvedCount, + contradictoryDirectChildren: contradictoryChildren.length, + confidenceCapReason, reason: "No direct child progress exists yet for the parent.", }; } @@ -635,6 +716,17 @@ export function propagateResolvedChildEvidence({ parentStatusAfter: null, parentConfidenceBefore: null, parentConfidenceAfter: null, + evidenceConfidenceBefore: null, + evidenceConfidenceAfter: null, + completenessBefore: null, + completenessAfter: null, + conclusionConfidenceBefore: null, + conclusionConfidenceAfter: null, + resolvedDirectChildren: 0, + unresolvedDirectChildren: 0, + contradictoryDirectChildren: 0, + confidenceCapReason: null, + ancestorPropagationStoppedReason: "no_resolved_child_propagation_needed", affectedAncestorIds: [], nextSelectedSibling: null, parentResolved: false, @@ -646,6 +738,7 @@ export function propagateResolvedChildEvidence({ syncParentChildReferences(graph); const propagationEvents = []; const affectedAncestorIds = new Set(); + let ancestorPropagationStoppedReason = "no_ancestor_state_changed"; for (const resolvedChildNode of resolvedChildNodes) { const liveChildNode = graph.nodes.find( @@ -662,10 +755,24 @@ export function propagateResolvedChildEvidence({ for (const ancestorNode of ancestorChain) { const beforeStatus = ancestorNode.status; const beforeConfidence = ancestorNode.confidence; + const beforeAssessment = getNodeConfidenceAssessment(ancestorNode); const progressState = computeParentProgressState(graph, ancestorNode); ancestorNode.status = progressState.nextStatus; ancestorNode.confidence = progressState.nextConfidence; + ancestorNode.confidenceAssessment = + progressState.nextConfidenceAssessment; + + if ( + beforeStatus === progressState.nextStatus && + beforeConfidence === progressState.nextConfidence && + JSON.stringify(beforeAssessment) === + JSON.stringify(progressState.nextConfidenceAssessment) + ) { + continue; + } + + ancestorPropagationStoppedReason = "ancestor_state_changed"; if (progressState.parentResolved) { ensureResolvedUnknownId(proposalSnapshot, ancestorNode.id); @@ -688,6 +795,19 @@ export function propagateResolvedChildEvidence({ parentStatusAfter: progressState.nextStatus, parentConfidenceBefore: beforeConfidence, parentConfidenceAfter: progressState.nextConfidence, + evidenceConfidenceBefore: beforeAssessment.evidenceConfidence, + evidenceConfidenceAfter: + progressState.nextConfidenceAssessment.evidenceConfidence, + completenessBefore: beforeAssessment.completenessStatus, + completenessAfter: + progressState.nextConfidenceAssessment.completenessStatus, + conclusionConfidenceBefore: beforeAssessment.conclusionConfidence, + conclusionConfidenceAfter: + progressState.nextConfidenceAssessment.conclusionConfidence, + resolvedDirectChildren: progressState.resolvedDirectChildren, + unresolvedDirectChildren: progressState.unresolvedDirectChildren, + contradictoryDirectChildren: progressState.contradictoryDirectChildren, + confidenceCapReason: progressState.confidenceCapReason, parentResolved: progressState.parentResolved, reason: progressState.reason, }); @@ -717,6 +837,17 @@ export function propagateResolvedChildEvidence({ parentStatusAfter: firstEvent?.parentStatusAfter ?? null, parentConfidenceBefore: firstEvent?.parentConfidenceBefore ?? null, parentConfidenceAfter: firstEvent?.parentConfidenceAfter ?? null, + evidenceConfidenceBefore: firstEvent?.evidenceConfidenceBefore ?? null, + evidenceConfidenceAfter: firstEvent?.evidenceConfidenceAfter ?? null, + completenessBefore: firstEvent?.completenessBefore ?? null, + completenessAfter: firstEvent?.completenessAfter ?? null, + conclusionConfidenceBefore: firstEvent?.conclusionConfidenceBefore ?? null, + conclusionConfidenceAfter: firstEvent?.conclusionConfidenceAfter ?? null, + resolvedDirectChildren: firstEvent?.resolvedDirectChildren ?? 0, + unresolvedDirectChildren: firstEvent?.unresolvedDirectChildren ?? 0, + contradictoryDirectChildren: firstEvent?.contradictoryDirectChildren ?? 0, + confidenceCapReason: firstEvent?.confidenceCapReason ?? null, + ancestorPropagationStoppedReason, affectedAncestorIds: [...affectedAncestorIds], nextSelectedSibling: siblingSelection?.status === "selected" ? siblingSelection.nodeId : null, @@ -1767,6 +1898,20 @@ export function applyValidatedProposal({ const parentStatusAfter = propagationResult.parentStatusAfter; const parentConfidenceBefore = propagationResult.parentConfidenceBefore; const parentConfidenceAfter = propagationResult.parentConfidenceAfter; + const evidenceConfidenceBefore = propagationResult.evidenceConfidenceBefore; + const evidenceConfidenceAfter = propagationResult.evidenceConfidenceAfter; + const completenessBefore = propagationResult.completenessBefore; + const completenessAfter = propagationResult.completenessAfter; + const conclusionConfidenceBefore = + propagationResult.conclusionConfidenceBefore; + const conclusionConfidenceAfter = propagationResult.conclusionConfidenceAfter; + const resolvedDirectChildren = propagationResult.resolvedDirectChildren; + const unresolvedDirectChildren = propagationResult.unresolvedDirectChildren; + const contradictoryDirectChildren = + propagationResult.contradictoryDirectChildren; + const confidenceCapReason = propagationResult.confidenceCapReason; + const ancestorPropagationStoppedReason = + propagationResult.ancestorPropagationStoppedReason; const affectedAncestorIds = propagationResult.affectedAncestorIds; const nextSelectedSibling = propagationResult.nextSelectedSibling; const parentResolved = propagationResult.parentResolved; @@ -1894,6 +2039,17 @@ export function applyValidatedProposal({ parentStatusAfter, parentConfidenceBefore, parentConfidenceAfter, + evidenceConfidenceBefore, + evidenceConfidenceAfter, + completenessBefore, + completenessAfter, + conclusionConfidenceBefore, + conclusionConfidenceAfter, + resolvedDirectChildren, + unresolvedDirectChildren, + contradictoryDirectChildren, + confidenceCapReason, + ancestorPropagationStoppedReason, affectedAncestorIds, nextSelectedSibling, parentResolved, diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index 4995d18..551ec03 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -110,6 +110,17 @@ function buildUpdateDiagnostics({ parentStatusAfter, parentConfidenceBefore, parentConfidenceAfter, + evidenceConfidenceBefore, + evidenceConfidenceAfter, + completenessBefore, + completenessAfter, + conclusionConfidenceBefore, + conclusionConfidenceAfter, + resolvedDirectChildren, + unresolvedDirectChildren, + contradictoryDirectChildren, + confidenceCapReason, + ancestorPropagationStoppedReason, affectedAncestorIds, nextSelectedSibling, parentResolved, @@ -163,6 +174,17 @@ function buildUpdateDiagnostics({ parentStatusAfter: parentStatusAfter ?? null, parentConfidenceBefore: parentConfidenceBefore ?? null, parentConfidenceAfter: parentConfidenceAfter ?? null, + evidenceConfidenceBefore: evidenceConfidenceBefore ?? null, + evidenceConfidenceAfter: evidenceConfidenceAfter ?? null, + completenessBefore: completenessBefore ?? null, + completenessAfter: completenessAfter ?? null, + conclusionConfidenceBefore: conclusionConfidenceBefore ?? null, + conclusionConfidenceAfter: conclusionConfidenceAfter ?? null, + resolvedDirectChildren: resolvedDirectChildren ?? 0, + unresolvedDirectChildren: unresolvedDirectChildren ?? 0, + contradictoryDirectChildren: contradictoryDirectChildren ?? 0, + confidenceCapReason: confidenceCapReason ?? null, + ancestorPropagationStoppedReason: ancestorPropagationStoppedReason ?? null, affectedAncestorIds: affectedAncestorIds ?? [], nextSelectedSibling: nextSelectedSibling ?? null, parentResolved: parentResolved ?? false, @@ -440,6 +462,17 @@ async function updateCaseWithDependencies(body, dependencies = {}) { parentStatusAfter: null, parentConfidenceBefore: null, parentConfidenceAfter: null, + evidenceConfidenceBefore: null, + evidenceConfidenceAfter: null, + completenessBefore: null, + completenessAfter: null, + conclusionConfidenceBefore: null, + conclusionConfidenceAfter: null, + resolvedDirectChildren: 0, + unresolvedDirectChildren: 0, + contradictoryDirectChildren: 0, + confidenceCapReason: null, + ancestorPropagationStoppedReason: null, affectedAncestorIds: [], nextSelectedSibling: null, parentResolved: false, @@ -508,6 +541,20 @@ async function updateCaseWithDependencies(body, dependencies = {}) { parentStatusAfter: applicationResult.parentStatusAfter, parentConfidenceBefore: applicationResult.parentConfidenceBefore, parentConfidenceAfter: applicationResult.parentConfidenceAfter, + evidenceConfidenceBefore: applicationResult.evidenceConfidenceBefore, + evidenceConfidenceAfter: applicationResult.evidenceConfidenceAfter, + completenessBefore: applicationResult.completenessBefore, + completenessAfter: applicationResult.completenessAfter, + conclusionConfidenceBefore: + applicationResult.conclusionConfidenceBefore, + conclusionConfidenceAfter: applicationResult.conclusionConfidenceAfter, + resolvedDirectChildren: applicationResult.resolvedDirectChildren, + unresolvedDirectChildren: applicationResult.unresolvedDirectChildren, + contradictoryDirectChildren: + applicationResult.contradictoryDirectChildren, + confidenceCapReason: applicationResult.confidenceCapReason, + ancestorPropagationStoppedReason: + applicationResult.ancestorPropagationStoppedReason, affectedAncestorIds: applicationResult.affectedAncestorIds, nextSelectedSibling: applicationResult.nextSelectedSibling, parentResolved: applicationResult.parentResolved, @@ -560,6 +607,17 @@ async function updateCaseWithDependencies(body, dependencies = {}) { parentStatusAfter: null, parentConfidenceBefore: null, parentConfidenceAfter: null, + evidenceConfidenceBefore: null, + evidenceConfidenceAfter: null, + completenessBefore: null, + completenessAfter: null, + conclusionConfidenceBefore: null, + conclusionConfidenceAfter: null, + resolvedDirectChildren: 0, + unresolvedDirectChildren: 0, + contradictoryDirectChildren: 0, + confidenceCapReason: null, + ancestorPropagationStoppedReason: null, affectedAncestorIds: [], nextSelectedSibling: null, parentResolved: false, diff --git a/lib/graph/schema.js b/lib/graph/schema.js index 680a5bc..fbe173a 100644 --- a/lib/graph/schema.js +++ b/lib/graph/schema.js @@ -36,6 +36,20 @@ export const ConfidenceLevel = /** @type {const} */ ({ high: "high", }); +export const CompletenessStatus = /** @type {const} */ ({ + empty: "empty", + partial: "partial", + complete: "complete", +}); + +export const confidenceAssessmentSchema = z + .object({ + evidenceConfidence: z.enum(Object.values(ConfidenceLevel)), + completenessStatus: z.enum(Object.values(CompletenessStatus)), + conclusionConfidence: z.enum(Object.values(ConfidenceLevel)), + }) + .strict(); + // ── SituationNode ──────────────────────────────────── export const situationNodeSchema = z.object({ @@ -45,6 +59,7 @@ export const situationNodeSchema = z.object({ kind: z.enum(Object.values(SituationKind)), status: z.enum(Object.values(SituationStatus)), confidence: z.enum(Object.values(ConfidenceLevel)), + confidenceAssessment: confidenceAssessmentSchema.optional(), value: z.union([z.string(), z.number(), z.null()]).nullable().optional(), unit: z.string().nullable().optional(), evidenceIds: z.array(z.string()).default([]), @@ -188,6 +203,7 @@ export function makeNode(opts) { kind: opts.kind ?? "observation", status: opts.status ?? "unknown", confidence: opts.confidence ?? "medium", + confidenceAssessment: opts.confidenceAssessment, value: opts.value ?? null, unit: opts.unit ?? null, evidenceIds: opts.evidenceIds ?? [], diff --git a/tests/graph/confidence-propagation.test.js b/tests/graph/confidence-propagation.test.js new file mode 100644 index 0000000..c8e6440 --- /dev/null +++ b/tests/graph/confidence-propagation.test.js @@ -0,0 +1,179 @@ +import { describe, expect, it } from "vitest"; +import { applyValidatedProposal } from "@/lib/graph/apply-proposal.js"; +import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js"; + +function makeFixture() { + const parent = makeNode({ + id: "n-parent", + label: "Explanation for why revenue increased while cash fell", + description: + "Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.", + kind: "unknown", + status: "unknown", + confidence: "medium", + }); + const children = [ + makeNode({ + id: "n-child-1", + label: "How the two observations were measured", + description: "Need evidence about the measure used for each observation.", + kind: "unknown", + status: "unknown", + confidence: "medium", + parentId: parent.id, + }), + makeNode({ + id: "n-child-2", + label: "Whether the two observations reflect different timing", + description: + "Need to know whether the two observations reflect different timing.", + kind: "unknown", + status: "unknown", + confidence: "medium", + parentId: parent.id, + }), + makeNode({ + id: "n-child-3", + label: "Possible change mainly affecting revenue", + description: + "Need to know whether a possible change mainly affected revenue.", + kind: "unknown", + status: "unknown", + confidence: "medium", + parentId: parent.id, + }), + makeNode({ + id: "n-child-4", + label: "Possible one-off event during the period", + description: + "Need to know whether a possible one-off event happened during the period.", + kind: "unknown", + status: "unknown", + confidence: "medium", + parentId: parent.id, + }), + ]; + + return makeGraph({ + centralStatement: "Revenue increased while cash fell.", + nodes: [parent, ...children], + edges: children.map((child, index) => + makeEdge({ + id: `e-${index + 1}`, + fromNodeId: child.id, + toNodeId: parent.id, + relationship: "depends_on", + description: `${child.label} feeds the parent.`, + }), + ), + activeUnknownNodeId: "n-child-1", + resolvedNodeIds: [], + currentSummary: "confidence propagation fixture", + }); +} + +function makeProposal({ resolvedIds, contradictedIds = [] }) { + return { + addedNodes: [ + makeNode({ + id: "n-anchor", + label: "Update anchor", + description: + "Anchor state introduced by the answer because the update must contain a meaningful change.", + kind: "state", + status: "known", + confidence: "low", + }), + ], + updatedNodes: [ + ...resolvedIds.map((id) => ({ + nodeId: id, + previousStatus: "unknown", + newStatus: "resolved", + previousValue: null, + newValue: `answer:${id}`, + reason: "resolved child", + })), + ...contradictedIds.map((id) => ({ + nodeId: id, + previousStatus: "unknown", + newStatus: "contradicted", + previousValue: null, + newValue: `contradiction:${id}`, + reason: "contradictory child evidence", + })), + ], + addedEdges: [], + removedEdgeIds: [], + resolvedUnknownNodeIds: resolvedIds, + affectedNodeIds: [], + selectedQuestion: null, + }; +} + +describe("confidence propagation", () => { + it("one of four children resolved does not yield high conclusion confidence", () => { + const result = applyValidatedProposal({ + situationGraph: makeFixture(), + proposal: makeProposal({ resolvedIds: ["n-child-1"] }), + previousQuestion: + "What evidence would clarify how the two observations were measured?", + answer: "Same accounting period and same management accounts.", + }); + + const parent = result.updatedSituationGraph.nodes.find( + (n) => n.id === "n-parent", + ); + expect(parent.status).toBe("provisional"); + expect(parent.confidence).toBe("medium"); + expect(parent.confidenceAssessment).toEqual({ + evidenceConfidence: "high", + completenessStatus: "partial", + conclusionConfidence: "medium", + }); + expect(result.confidenceCapReason).toBe( + "unresolved_direct_children_cap_conclusion", + ); + }); + + it("all children resolved with coherent evidence may yield high confidence", () => { + const result = applyValidatedProposal({ + situationGraph: makeFixture(), + proposal: makeProposal({ + resolvedIds: ["n-child-1", "n-child-2", "n-child-3", "n-child-4"], + }), + previousQuestion: + "What evidence would clarify how the two observations were measured?", + answer: "All direct child questions are answered.", + }); + + const parent = result.updatedSituationGraph.nodes.find( + (n) => n.id === "n-parent", + ); + expect(parent.status).toBe("resolved"); + expect(parent.confidenceAssessment).toEqual({ + evidenceConfidence: "high", + completenessStatus: "complete", + conclusionConfidence: "high", + }); + }); + + it("contradictory child evidence prevents high confidence", () => { + const result = applyValidatedProposal({ + situationGraph: makeFixture(), + proposal: makeProposal({ + resolvedIds: ["n-child-1"], + contradictedIds: ["n-child-2"], + }), + previousQuestion: + "What evidence would clarify how the two observations were measured?", + answer: "One child resolved, another contradicted.", + }); + + const parent = result.updatedSituationGraph.nodes.find( + (n) => n.id === "n-parent", + ); + expect(parent.confidenceAssessment.conclusionConfidence).toBe("low"); + expect(result.confidenceCapReason).toBe("contradictory_direct_children"); + }); +}); diff --git a/tests/graph/upward-propagation.test.js b/tests/graph/upward-propagation.test.js index 2e6731e..ed0658f 100644 --- a/tests/graph/upward-propagation.test.js +++ b/tests/graph/upward-propagation.test.js @@ -255,7 +255,16 @@ describe("upward propagation", () => { expect(result.parentStatusBefore).toBe("unknown"); expect(result.parentStatusAfter).toBe("provisional"); expect(result.parentConfidenceBefore).toBe("medium"); - expect(result.parentConfidenceAfter).toBe("high"); + expect(result.parentConfidenceAfter).toBe("medium"); + expect(result.evidenceConfidenceBefore).toBe("medium"); + expect(result.evidenceConfidenceAfter).toBe("high"); + expect(result.completenessBefore).toBe("empty"); + expect(result.completenessAfter).toBe("partial"); + expect(result.conclusionConfidenceBefore).toBe("low"); + expect(result.conclusionConfidenceAfter).toBe("medium"); + expect(result.confidenceCapReason).toBe( + "unresolved_direct_children_cap_conclusion", + ); expect(result.parentResolved).toBe(false); expect(result.affectedAncestorIds).toContain(ids.parent); expect(result.affectedAncestorIds).toContain(ids.ancestor); @@ -276,7 +285,12 @@ describe("upward propagation", () => { ); expect(parentNode).toMatchObject({ status: "provisional", - confidence: "high", + confidence: "medium", + confidenceAssessment: { + evidenceConfidence: "high", + completenessStatus: "partial", + conclusionConfidence: "medium", + }, }); const ancestorNode = result.updatedSituationGraph.nodes.find( @@ -284,7 +298,12 @@ describe("upward propagation", () => { ); expect(ancestorNode).toMatchObject({ status: "provisional", - confidence: "high", + confidence: "low", + confidenceAssessment: { + evidenceConfidence: "low", + completenessStatus: "empty", + conclusionConfidence: "low", + }, }); const resolvedChild = result.updatedSituationGraph.nodes.find( @@ -387,6 +406,14 @@ describe("upward propagation", () => { expect(result.resolvedUnknownNodeIds).toContain(ids.parent); expect( result.updatedSituationGraph.nodes.find((node) => node.id === ids.parent), - ).toMatchObject({ status: "resolved", confidence: "high" }); + ).toMatchObject({ + status: "resolved", + confidence: "high", + confidenceAssessment: { + evidenceConfidence: "high", + completenessStatus: "complete", + conclusionConfidence: "high", + }, + }); }); }); From b2ffc5496417f29e0ea8a60a43d37f99c74f8fc4 Mon Sep 17 00:00:00 2001 From: robbond Date: Mon, 3 Aug 2026 07:19:20 +0100 Subject: [PATCH 15/17] feat: evaluate deterministic cross-branch corroboration --- docs/v0.6-atomicity-experiment.md | 34 ++ lib/graph/apply-proposal.js | 183 ++++++++++- lib/graph/orchestrator.js | 25 ++ tests/graph/confidence-propagation.test.js | 2 +- .../graph/cross-branch-corroboration.test.js | 306 ++++++++++++++++++ tests/graph/upward-propagation.test.js | 4 +- 6 files changed, 547 insertions(+), 7 deletions(-) create mode 100644 tests/graph/cross-branch-corroboration.test.js diff --git a/docs/v0.6-atomicity-experiment.md b/docs/v0.6-atomicity-experiment.md index 6c5ce96..c945112 100644 --- a/docs/v0.6-atomicity-experiment.md +++ b/docs/v0.6-atomicity-experiment.md @@ -149,6 +149,40 @@ Deterministic propagation rules now enforce: - status changes do not raise confidence on their own - parent resolution still requires the separate completion rule +## Cross-branch corroboration + +The next confidence experiment adds deterministic branch interaction checks without changing the graph model. + +The engine now distinguishes between: + +- **multiple evidence**: more than one branch exists +- **independent corroboration**: distinct resolved branches support the same parent without sharing the same evidence key +- **duplicate evidence**: the same evidence key appears through multiple branches and must not be double-counted +- **conflicting evidence**: branches support incompatible positions, such as `recognised correctly` vs `recognised incorrectly` + +Deterministic branch rules: + +- corroboration only counts when branches are distinct and their evidence sources differ +- duplicate evidence groups never count as corroboration +- conflicts cap conclusion confidence and prevent a higher confidence upgrade +- independent branches remain interaction-neutral + +Additional diagnostics now expose: + +- `corroboratingBranchCount` +- `conflictingBranchCount` +- `duplicateEvidenceCount` +- `independentBranchCount` +- `interactionSummary` +- `confidenceAdjustmentReason` + +Observed effect: + +- independent corroboration can raise `evidenceConfidence` +- duplicate evidence produces no extra confidence increase +- conflicting evidence lowers or caps `conclusionConfidence` +- completeness rules still dominate whether a parent may become highly justified + Example progression: - parent before: `unknown`, `medium` diff --git a/lib/graph/apply-proposal.js b/lib/graph/apply-proposal.js index 98dfa5b..67be4d1 100644 --- a/lib/graph/apply-proposal.js +++ b/lib/graph/apply-proposal.js @@ -463,6 +463,129 @@ function confidenceFromAssessment(assessment) { return assessment?.conclusionConfidence ?? "medium"; } +function unique(values = []) { + return [...new Set(values.filter(Boolean))]; +} + +function branchEvidenceKeys(node) { + return unique([...(node?.evidenceIds || []), node?.value]); +} + +function sharedMeaningfulTokens(aText, bText) { + const stop = new Set([ + "the", + "and", + "for", + "that", + "this", + "with", + "from", + "because", + "need", + "unknown", + "possible", + ]); + const a = splitSemanticTokens(aText).filter((token) => !stop.has(token)); + const b = splitSemanticTokens(bText).filter((token) => !stop.has(token)); + return [...new Set(a.filter((token) => b.includes(token)))]; +} + +function branchConflictSignature(node) { + return normaliseText( + `${node?.label || ""} ${node?.description || ""} ${node?.value || ""}`, + ); +} + +function branchesConflict(aNode, bNode) { + const aText = branchConflictSignature(aNode); + const bText = branchConflictSignature(bNode); + const oppositePolarity = + (aText.includes("correctly") && bText.includes("incorrectly")) || + (aText.includes("incorrectly") && bText.includes("correctly")) || + aNode?.status === "contradicted" || + bNode?.status === "contradicted"; + + if (!oppositePolarity) return false; + + return sharedMeaningfulTokens(aText, bText).length >= 2; +} + +export function evaluateBranchInteractions({ parentNode, graph }) { + const directBranches = findDirectChildUnknowns(graph, parentNode.id).filter( + (node) => ["resolved", "provisional", "contradicted"].includes(node.status), + ); + const duplicateEvidenceGroups = []; + const conflictingBranches = []; + const corroboratingBranches = []; + const duplicateBranchIds = new Set(); + const conflictingBranchIds = new Set(); + + const evidenceGroups = new Map(); + for (const branch of directBranches) { + for (const evidenceKey of branchEvidenceKeys(branch)) { + const ids = evidenceGroups.get(evidenceKey) || []; + ids.push(branch.id); + evidenceGroups.set(evidenceKey, ids); + } + } + + for (const [evidenceKey, branchIds] of evidenceGroups.entries()) { + if (branchIds.length > 1) { + duplicateEvidenceGroups.push({ + evidenceKey, + branchIds: unique(branchIds), + }); + for (const id of branchIds) duplicateBranchIds.add(id); + } + } + + for (let index = 0; index < directBranches.length; index += 1) { + for (let inner = index + 1; inner < directBranches.length; inner += 1) { + const aNode = directBranches[index]; + const bNode = directBranches[inner]; + if (branchesConflict(aNode, bNode)) { + conflictingBranches.push([aNode.id, bNode.id]); + conflictingBranchIds.add(aNode.id); + conflictingBranchIds.add(bNode.id); + continue; + } + + const aEvidence = branchEvidenceKeys(aNode); + const bEvidence = branchEvidenceKeys(bNode); + const sharesEvidence = aEvidence.some((key) => bEvidence.includes(key)); + if ( + !sharesEvidence && + aNode.status === "resolved" && + bNode.status === "resolved" + ) { + corroboratingBranches.push([aNode.id, bNode.id]); + } + } + } + + const interactionBranchIds = new Set([ + ...duplicateBranchIds, + ...conflictingBranchIds, + ...corroboratingBranches.flat(), + ]); + const independentBranches = directBranches + .map((branch) => branch.id) + .filter((id) => !interactionBranchIds.has(id)); + + return { + corroboratingBranches, + conflictingBranches, + duplicateEvidenceGroups, + independentBranches, + interactionSummary: { + corroboratingBranchCount: corroboratingBranches.length, + conflictingBranchCount: conflictingBranches.length, + duplicateEvidenceCount: duplicateEvidenceGroups.length, + independentBranchCount: independentBranches.length, + }, + }; +} + function upsertProposalNodeUpdate(proposalSnapshot, update) { const existing = proposalSnapshot.updatedNodes.find( (candidate) => candidate.nodeId === update.nodeId, @@ -585,6 +708,13 @@ function computeParentProgressState(graph, parentNode) { (child) => child.status !== "resolved", ).length; const beforeAssessment = getNodeConfidenceAssessment(parentNode); + const interactions = evaluateBranchInteractions({ parentNode, graph }); + const corroborationCount = + interactions.interactionSummary.corroboratingBranchCount; + const duplicateEvidenceCount = + interactions.interactionSummary.duplicateEvidenceCount; + const conflictingBranchCount = + interactions.interactionSummary.conflictingBranchCount; if (totalChildren === 0) { const nextAssessment = { @@ -625,18 +755,32 @@ function computeParentProgressState(graph, parentNode) { confidenceCapReason = "no_resolved_direct_children"; } else if (resolvedCount < totalChildren) { nextAssessment = { - evidenceConfidence: "high", + evidenceConfidence: corroborationCount > 0 ? "high" : "medium", completenessStatus: "partial", conclusionConfidence: "medium", }; - confidenceCapReason = "unresolved_direct_children_cap_conclusion"; + confidenceCapReason = + conflictingBranchCount > 0 + ? "conflicting_branches_cap_conclusion" + : duplicateEvidenceCount > 0 + ? "duplicate_evidence_no_extra_confidence" + : corroborationCount > 0 + ? "independent_corroboration_with_incomplete_parent" + : "unresolved_direct_children_cap_conclusion"; } else { nextAssessment = { evidenceConfidence: "high", completenessStatus: "complete", - conclusionConfidence: "high", + conclusionConfidence: conflictingBranchCount > 0 ? "low" : "high", }; - confidenceCapReason = null; + confidenceCapReason = + conflictingBranchCount > 0 + ? "conflicting_branches_cap_conclusion" + : duplicateEvidenceCount > 0 + ? "duplicate_evidence_no_extra_confidence" + : corroborationCount > 0 + ? "independent_corroboration_supported_conclusion" + : null; } if (resolvedChildren.length === totalChildren) { @@ -652,6 +796,7 @@ function computeParentProgressState(graph, parentNode) { resolvedDirectChildren: resolvedCount, unresolvedDirectChildren: unresolvedCount, contradictoryDirectChildren: contradictoryChildren.length, + branchInteractions: interactions, confidenceCapReason, reason: "All direct child unknowns are resolved, so the parent can now resolve deterministically.", @@ -671,6 +816,7 @@ function computeParentProgressState(graph, parentNode) { resolvedDirectChildren: resolvedCount, unresolvedDirectChildren: unresolvedCount, contradictoryDirectChildren: contradictoryChildren.length, + branchInteractions: interactions, confidenceCapReason, reason: "At least one direct child has been progressed, so the parent becomes provisional but remains unresolved until all direct children are resolved.", @@ -689,6 +835,7 @@ function computeParentProgressState(graph, parentNode) { resolvedDirectChildren: resolvedCount, unresolvedDirectChildren: unresolvedCount, contradictoryDirectChildren: contradictoryChildren.length, + branchInteractions: interactions, confidenceCapReason, reason: "No direct child progress exists yet for the parent.", }; @@ -807,6 +954,19 @@ export function propagateResolvedChildEvidence({ resolvedDirectChildren: progressState.resolvedDirectChildren, unresolvedDirectChildren: progressState.unresolvedDirectChildren, contradictoryDirectChildren: progressState.contradictoryDirectChildren, + corroboratingBranchCount: + progressState.branchInteractions.interactionSummary + .corroboratingBranchCount, + conflictingBranchCount: + progressState.branchInteractions.interactionSummary + .conflictingBranchCount, + duplicateEvidenceCount: + progressState.branchInteractions.interactionSummary + .duplicateEvidenceCount, + independentBranchCount: + progressState.branchInteractions.interactionSummary + .independentBranchCount, + interactionSummary: progressState.branchInteractions.interactionSummary, confidenceCapReason: progressState.confidenceCapReason, parentResolved: progressState.parentResolved, reason: progressState.reason, @@ -846,6 +1006,11 @@ export function propagateResolvedChildEvidence({ resolvedDirectChildren: firstEvent?.resolvedDirectChildren ?? 0, unresolvedDirectChildren: firstEvent?.unresolvedDirectChildren ?? 0, contradictoryDirectChildren: firstEvent?.contradictoryDirectChildren ?? 0, + corroboratingBranchCount: firstEvent?.corroboratingBranchCount ?? 0, + conflictingBranchCount: firstEvent?.conflictingBranchCount ?? 0, + duplicateEvidenceCount: firstEvent?.duplicateEvidenceCount ?? 0, + independentBranchCount: firstEvent?.independentBranchCount ?? 0, + interactionSummary: firstEvent?.interactionSummary ?? null, confidenceCapReason: firstEvent?.confidenceCapReason ?? null, ancestorPropagationStoppedReason, affectedAncestorIds: [...affectedAncestorIds], @@ -1915,6 +2080,11 @@ export function applyValidatedProposal({ const affectedAncestorIds = propagationResult.affectedAncestorIds; const nextSelectedSibling = propagationResult.nextSelectedSibling; const parentResolved = propagationResult.parentResolved; + const corroboratingBranchCount = propagationResult.corroboratingBranchCount; + const conflictingBranchCount = propagationResult.conflictingBranchCount; + const duplicateEvidenceCount = propagationResult.duplicateEvidenceCount; + const independentBranchCount = propagationResult.independentBranchCount; + const interactionSummary = propagationResult.interactionSummary; const propagationReason = propagationResult.reason; if ( @@ -2048,6 +2218,11 @@ export function applyValidatedProposal({ resolvedDirectChildren, unresolvedDirectChildren, contradictoryDirectChildren, + corroboratingBranchCount, + conflictingBranchCount, + duplicateEvidenceCount, + independentBranchCount, + interactionSummary, confidenceCapReason, ancestorPropagationStoppedReason, affectedAncestorIds, diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index 551ec03..d09ae7b 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -119,6 +119,11 @@ function buildUpdateDiagnostics({ resolvedDirectChildren, unresolvedDirectChildren, contradictoryDirectChildren, + corroboratingBranchCount, + conflictingBranchCount, + duplicateEvidenceCount, + independentBranchCount, + interactionSummary, confidenceCapReason, ancestorPropagationStoppedReason, affectedAncestorIds, @@ -183,6 +188,11 @@ function buildUpdateDiagnostics({ resolvedDirectChildren: resolvedDirectChildren ?? 0, unresolvedDirectChildren: unresolvedDirectChildren ?? 0, contradictoryDirectChildren: contradictoryDirectChildren ?? 0, + corroboratingBranchCount: corroboratingBranchCount ?? 0, + conflictingBranchCount: conflictingBranchCount ?? 0, + duplicateEvidenceCount: duplicateEvidenceCount ?? 0, + independentBranchCount: independentBranchCount ?? 0, + interactionSummary: interactionSummary ?? null, confidenceCapReason: confidenceCapReason ?? null, ancestorPropagationStoppedReason: ancestorPropagationStoppedReason ?? null, affectedAncestorIds: affectedAncestorIds ?? [], @@ -471,6 +481,11 @@ async function updateCaseWithDependencies(body, dependencies = {}) { resolvedDirectChildren: 0, unresolvedDirectChildren: 0, contradictoryDirectChildren: 0, + corroboratingBranchCount: 0, + conflictingBranchCount: 0, + duplicateEvidenceCount: 0, + independentBranchCount: 0, + interactionSummary: null, confidenceCapReason: null, ancestorPropagationStoppedReason: null, affectedAncestorIds: [], @@ -552,6 +567,11 @@ async function updateCaseWithDependencies(body, dependencies = {}) { unresolvedDirectChildren: applicationResult.unresolvedDirectChildren, contradictoryDirectChildren: applicationResult.contradictoryDirectChildren, + corroboratingBranchCount: applicationResult.corroboratingBranchCount, + conflictingBranchCount: applicationResult.conflictingBranchCount, + duplicateEvidenceCount: applicationResult.duplicateEvidenceCount, + independentBranchCount: applicationResult.independentBranchCount, + interactionSummary: applicationResult.interactionSummary, confidenceCapReason: applicationResult.confidenceCapReason, ancestorPropagationStoppedReason: applicationResult.ancestorPropagationStoppedReason, @@ -616,6 +636,11 @@ async function updateCaseWithDependencies(body, dependencies = {}) { resolvedDirectChildren: 0, unresolvedDirectChildren: 0, contradictoryDirectChildren: 0, + corroboratingBranchCount: 0, + conflictingBranchCount: 0, + duplicateEvidenceCount: 0, + independentBranchCount: 0, + interactionSummary: null, confidenceCapReason: null, ancestorPropagationStoppedReason: null, affectedAncestorIds: [], diff --git a/tests/graph/confidence-propagation.test.js b/tests/graph/confidence-propagation.test.js index c8e6440..6804d06 100644 --- a/tests/graph/confidence-propagation.test.js +++ b/tests/graph/confidence-propagation.test.js @@ -127,7 +127,7 @@ describe("confidence propagation", () => { expect(parent.status).toBe("provisional"); expect(parent.confidence).toBe("medium"); expect(parent.confidenceAssessment).toEqual({ - evidenceConfidence: "high", + evidenceConfidence: "medium", completenessStatus: "partial", conclusionConfidence: "medium", }); diff --git a/tests/graph/cross-branch-corroboration.test.js b/tests/graph/cross-branch-corroboration.test.js new file mode 100644 index 0000000..b823ab2 --- /dev/null +++ b/tests/graph/cross-branch-corroboration.test.js @@ -0,0 +1,306 @@ +import { describe, expect, it } from "vitest"; +import { + applyValidatedProposal, + evaluateBranchInteractions, +} from "@/lib/graph/apply-proposal.js"; +import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js"; + +function makeParentWithBranches(children) { + const parent = makeNode({ + id: "n-parent", + label: "Explanation for why revenue increased while cash fell", + description: + "Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.", + kind: "unknown", + status: "unknown", + confidence: "medium", + }); + + return makeGraph({ + centralStatement: "Revenue increased while cash fell.", + nodes: [ + parent, + ...children.map((child) => ({ ...child, parentId: parent.id })), + ], + edges: children.map((child, index) => + makeEdge({ + id: `e-${index + 1}`, + fromNodeId: child.id, + toNodeId: parent.id, + relationship: "depends_on", + description: `${child.label} feeds the parent.`, + }), + ), + activeUnknownNodeId: children[0]?.id ?? null, + resolvedNodeIds: [], + currentSummary: "cross-branch corroboration fixture", + }); +} + +function makeResolvedChild(id, label, value, extra = {}) { + return makeNode({ + id, + label, + description: label, + kind: "unknown", + status: "resolved", + confidence: "medium", + value, + evidenceIds: extra.evidenceIds ?? [], + }); +} + +function makeUnknownBranch(id, label, description, extra = {}) { + return makeNode({ + id, + label, + description, + kind: "unknown", + status: extra.status ?? "unknown", + confidence: extra.confidence ?? "medium", + evidenceIds: extra.evidenceIds ?? [], + value: extra.value ?? null, + }); +} + +describe("evaluateBranchInteractions", () => { + it("detects corroborating independent branches", () => { + const graph = makeParentWithBranches([ + makeResolvedChild("n-a", "Debtor balance increased", "bank-statement-a", { + evidenceIds: ["bank-statement-a"], + }), + makeResolvedChild( + "n-b", + "Cash receipts were delayed", + "receipts-ledger-b", + { evidenceIds: ["receipts-ledger-b"] }, + ), + ]); + const parentNode = graph.nodes.find((node) => node.id === "n-parent"); + + const result = evaluateBranchInteractions({ parentNode, graph }); + + expect(result.interactionSummary.corroboratingBranchCount).toBe(1); + expect(result.interactionSummary.duplicateEvidenceCount).toBe(0); + expect(result.interactionSummary.conflictingBranchCount).toBe(0); + }); + + it("detects duplicate evidence instead of corroboration", () => { + const graph = makeParentWithBranches([ + makeResolvedChild( + "n-a", + "Bank statement shows increased debtor balance", + "same-bank", + { + evidenceIds: ["same-bank"], + }, + ), + makeResolvedChild( + "n-b", + "Delayed receipts also cite the bank statement", + "same-bank", + { + evidenceIds: ["same-bank"], + }, + ), + ]); + const parentNode = graph.nodes.find((node) => node.id === "n-parent"); + + const result = evaluateBranchInteractions({ parentNode, graph }); + + expect(result.interactionSummary.duplicateEvidenceCount).toBe(1); + expect(result.interactionSummary.corroboratingBranchCount).toBe(0); + }); + + it("detects conflicting branches", () => { + const graph = makeParentWithBranches([ + makeResolvedChild("n-a", "Revenue recognised correctly", "correctly"), + makeNode({ + id: "n-b", + label: "Revenue recognised incorrectly", + description: "Revenue recognised incorrectly", + kind: "unknown", + status: "contradicted", + confidence: "medium", + value: "incorrectly", + }), + ]); + const parentNode = graph.nodes.find((node) => node.id === "n-parent"); + + const result = evaluateBranchInteractions({ parentNode, graph }); + + expect(result.interactionSummary.conflictingBranchCount).toBe(1); + }); +}); + +describe("cross-branch corroboration effects", () => { + function applyToGraph(children, resolvedIds, contradictedIds = []) { + const graph = makeParentWithBranches(children); + return applyValidatedProposal({ + situationGraph: graph, + proposal: { + addedNodes: [ + makeNode({ + id: "n-anchor", + label: "Update anchor", + description: + "Anchor state introduced by the answer because the update must contain a meaningful change.", + kind: "state", + status: "known", + confidence: "low", + }), + ], + updatedNodes: [ + ...resolvedIds.map((id) => ({ + nodeId: id, + previousStatus: "unknown", + newStatus: "resolved", + previousValue: null, + newValue: `answer:${id}`, + reason: "resolved child", + })), + ...contradictedIds.map((id) => ({ + nodeId: id, + previousStatus: "unknown", + newStatus: "contradicted", + previousValue: null, + newValue: `contradiction:${id}`, + reason: "contradicted child", + })), + ], + addedEdges: [], + removedEdgeIds: [], + resolvedUnknownNodeIds: resolvedIds, + affectedNodeIds: [], + selectedQuestion: null, + }, + previousQuestion: "What evidence would clarify this branch?", + answer: "deterministic branch update", + }); + } + + it("independent corroboration increases justified confidence without reaching high on incomplete parent", () => { + const result = applyToGraph( + [ + makeUnknownBranch( + "n-a", + "Debtor balance increased", + "Debtor balance increased", + ), + makeUnknownBranch( + "n-b", + "Cash receipts delayed", + "Cash receipts delayed", + ), + makeUnknownBranch( + "n-c", + "Possible one-off event during the period", + "Possible one-off event during the period", + ), + makeUnknownBranch( + "n-d", + "Whether the two observations reflect different timing", + "Whether the two observations reflect different timing", + ), + ], + ["n-a", "n-b"], + ); + + expect(result.success).toBe(true); + expect(result.interactionSummary?.corroboratingBranchCount).toBeGreaterThan( + 0, + ); + expect(result.interactionSummary?.duplicateEvidenceCount).toBe(0); + expect(result.parentConfidenceAfter).toBe("medium"); + expect(result.confidenceCapReason).toBe( + "independent_corroboration_with_incomplete_parent", + ); + }); + + it("duplicate evidence does not increase confidence", () => { + const result = applyToGraph( + [ + makeUnknownBranch( + "n-a", + "Bank statement shows increased debtor balance", + "Bank statement shows increased debtor balance", + { evidenceIds: ["same-bank"] }, + ), + makeUnknownBranch( + "n-b", + "Delayed receipts also cite the bank statement", + "Delayed receipts also cite the bank statement", + { evidenceIds: ["same-bank"] }, + ), + makeUnknownBranch( + "n-c", + "Possible one-off event during the period", + "Possible one-off event during the period", + ), + ], + ["n-a", "n-b"], + ); + + expect(result.success).toBe(true); + expect(result.interactionSummary?.duplicateEvidenceCount).toBeGreaterThan( + 0, + ); + expect(result.interactionSummary?.corroboratingBranchCount).toBe(0); + expect(result.confidenceCapReason).toBe( + "duplicate_evidence_no_extra_confidence", + ); + }); + + it("conflicting evidence caps confidence", () => { + const result = applyToGraph( + [ + makeUnknownBranch( + "n-a", + "Revenue recognised correctly", + "Revenue recognised correctly", + ), + makeUnknownBranch( + "n-b", + "Revenue recognised incorrectly", + "Revenue recognised incorrectly", + ), + makeUnknownBranch( + "n-c", + "Possible one-off event during the period", + "Possible one-off event during the period", + ), + ], + ["n-a"], + ["n-b"], + ); + + expect(result.success).toBe(true); + expect(result.interactionSummary?.conflictingBranchCount).toBeGreaterThan( + 0, + ); + expect(result.conclusionConfidenceAfter).toBe("low"); + }); + + it("independent branches stay interaction-neutral", () => { + const result = applyToGraph( + [ + makeUnknownBranch( + "n-a", + "Marketing campaign changed traffic", + "Marketing campaign changed traffic", + ), + makeUnknownBranch( + "n-b", + "Equipment maintenance occurred", + "Equipment maintenance occurred", + ), + ], + ["n-a"], + ); + + expect(result.success).toBe(true); + expect(result.interactionSummary?.independentBranchCount).toBeGreaterThan( + 0, + ); + }); +}); diff --git a/tests/graph/upward-propagation.test.js b/tests/graph/upward-propagation.test.js index ed0658f..a1545fe 100644 --- a/tests/graph/upward-propagation.test.js +++ b/tests/graph/upward-propagation.test.js @@ -257,7 +257,7 @@ describe("upward propagation", () => { expect(result.parentConfidenceBefore).toBe("medium"); expect(result.parentConfidenceAfter).toBe("medium"); expect(result.evidenceConfidenceBefore).toBe("medium"); - expect(result.evidenceConfidenceAfter).toBe("high"); + expect(result.evidenceConfidenceAfter).toBe("medium"); expect(result.completenessBefore).toBe("empty"); expect(result.completenessAfter).toBe("partial"); expect(result.conclusionConfidenceBefore).toBe("low"); @@ -287,7 +287,7 @@ describe("upward propagation", () => { status: "provisional", confidence: "medium", confidenceAssessment: { - evidenceConfidence: "high", + evidenceConfidence: "medium", completenessStatus: "partial", conclusionConfidence: "medium", }, From e0d9019c2a2b1db37401c65f172990ad7819de8f Mon Sep 17 00:00:00 2001 From: robbond Date: Mon, 3 Aug 2026 07:24:46 +0100 Subject: [PATCH 16/17] docs: document v0.6 reasoning architecture --- docs/v0.6-reasoning-architecture.md | 375 ++++++++++++++++++++++++++++ 1 file changed, 375 insertions(+) create mode 100644 docs/v0.6-reasoning-architecture.md diff --git a/docs/v0.6-reasoning-architecture.md b/docs/v0.6-reasoning-architecture.md new file mode 100644 index 0000000..0c456d6 --- /dev/null +++ b/docs/v0.6-reasoning-architecture.md @@ -0,0 +1,375 @@ +# v0.6 Reasoning Architecture + +## Purpose + +This document describes the implemented deterministic reasoning architecture on branch `feature/question-strategy-alignment-v0.6`. + +It is written for future developers who need to understand how v0.6 actually executes, what invariants it relies on, where the recursive loops are, and what the system deliberately does **not** attempt to do. + +## End-to-end pipeline + +The implemented runtime pipeline is: + +```text +Scenario input +↓ +LLM analysis / reconstruction +↓ +Initial graph build +↓ +Deterministic unknown selection +↓ +Selected question +↓ +User answer +↓ +LLM graph-update proposal +↓ +Proposal parsing / normalisation +↓ +Proposal compatibility validation +↓ +Deterministic graph update application +↓ +Reasoning-state rebuild +↓ +Comparability assessment +↓ +Relationship classification +↓ +Explicit emergent unknown creation / reuse (if required) +↓ +Deterministic reselection +↓ +Atomicity assessment +↓ +Optional decomposition into child unknowns +↓ +Deterministic reselection +↓ +Resolved-child propagation upward +↓ +Confidence / completeness / corroboration update +↓ +Next active unknown +↓ +Question formulation +``` + +## Deterministic stages + +### 1. Scenario analysis / reconstruction + +- **Purpose**: obtain structured reconstruction material from scenario text +- **Input**: scenario, prompt version +- **Output**: analysis payload containing reconstruction, evidence, diagnostics, and optional next question +- **Why it exists**: provides the initial structured substrate from which the graph is built +- **What breaks if removed**: the graph builder has no structured reconstruction to convert into nodes and edges + +### 2. Initial graph build + +- **Purpose**: convert reconstruction output into an initial `SituationGraph` +- **Input**: reconstruction + evidence +- **Output**: graph nodes and edges, then `makeGraph(...)` wraps them with active/resolved/summary state +- **Why it exists**: all later reasoning is graph-based, not free text +- **What breaks if removed**: no explicit unknown nodes, no deterministic selection, no validated update loop + +### 3. Deterministic unknown selection + +- **Purpose**: choose the next active unknown from unresolved graph nodes +- **Input**: graph, resolved node IDs +- **Output**: selected candidate or explicit ambiguity result +- **Why it exists**: the system needs a deterministic next investigation target +- **What breaks if removed**: question ordering becomes arbitrary or hidden in prompts + +### 4. Selected question exposure + +- **Purpose**: expose the chosen unknown as the next question to the user +- **Input**: selected unknown + question formulation or tie-resolution logic +- **Output**: selected question object +- **Why it exists**: the user-facing loop must ask a concrete next question +- **What breaks if removed**: the system can build a graph but cannot continue interaction coherently + +### 5. LLM graph-update proposal + +- **Purpose**: transform a user answer into a proposed graph change set +- **Input**: current graph, previous question, answer, prompt version +- **Output**: raw JSON-like proposal +- **Why it exists**: the LLM is limited to proposing changes; it does not mutate the graph directly +- **What breaks if removed**: answers cannot affect the graph except through manual hard-coded logic + +### 6. Proposal parsing / normalisation + +- **Purpose**: parse JSON, remove null array items, apply known aliases, fill omitted optional fields +- **Input**: raw model response +- **Output**: validated `graphUpdateSchema` payload or structured parser failure +- **Why it exists**: model outputs are not trusted as-is +- **What breaks if removed**: malformed or partially missing model output would reach graph logic directly + +### 7. Proposal compatibility validation + +- **Purpose**: ensure the proposal is graph-safe and semantically valid before application +- **Input**: current graph + proposed update +- **Output**: accepted proposal or compatibility errors +- **Why it exists**: protects graph integrity and reasoning invariants +- **What breaks if removed**: duplicate IDs, missing references, fake selected questions, and no-op updates could corrupt the graph + +### 8. Deterministic graph update application + +- **Purpose**: apply only validated graph changes to a copied graph +- **Input**: graph + validated proposal +- **Output**: updated nodes, edges, resolved node IDs +- **Why it exists**: separates safe application from generation +- **What breaks if removed**: no explicit, replayable state transition exists + +### 9. Reasoning-state rebuild + +- **Purpose**: derive fresh comparability/relationship state from the updated graph +- **Input**: updated graph + optional override state +- **Output**: `reasoningState` +- **Why it exists**: reasoning stages are derived from graph state, not stored blindly +- **What breaks if removed**: comparability and relationship decisions drift from actual graph contents + +### 10. Comparability assessment + +- **Purpose**: decide whether supported observations are comparable enough for relationship reasoning +- **Input**: graph observations + central statement + optional stored override +- **Output**: comparability status/reason + contradiction permission +- **Why it exists**: relationship reasoning is gated by comparability +- **What breaks if removed**: contradiction or relationship reasoning would run over incomparable observations + +### 11. Relationship classification + +- **Purpose**: classify observation relationships once comparability permits it +- **Input**: graph + comparability result +- **Output**: relationship status, reason, whether a follow-up question is justified +- **Why it exists**: determines whether explanation-style follow-up is needed +- **What breaks if removed**: the system cannot distinguish compatible, duplicate, insufficient, and contradiction-adjacent observation sets + +### 12. Explicit emergent unknown creation / reuse + +- **Purpose**: ensure any justified relationship follow-up is represented by an explicit unresolved graph node +- **Input**: provisional graph + relationship assessment +- **Output**: reused or newly added explanation unknown and edges +- **Why it exists**: preserves the invariant that a question must originate from an explicit unknown +- **What breaks if removed**: relationship follow-up would revert to fallback-only question text not backed by the graph + +### 13. Atomicity assessment + +- **Purpose**: determine whether the selected unknown is directly investigable or too composite +- **Input**: selected unknown + graph context +- **Output**: `atomic` or `composite` decision with decomposition kind/reason +- **Why it exists**: prevents asking broad explanation unknowns directly +- **What breaks if removed**: the system asks high-level composite unknowns instead of decomposing them first + +### 14. Optional decomposition + +- **Purpose**: split a composite unknown into deterministic child unknowns +- **Input**: composite selected unknown + graph context +- **Output**: 2–5 child unknowns, edges, quality summary, rejection diagnostics +- **Why it exists**: narrows broad unknowns into explicit candidate dimensions +- **What breaks if removed**: recursive reasoning stops at broad parents and loses graph-backed substructure + +### 15. Resolved-child propagation upward + +- **Purpose**: move resolved child effects to parent and ancestor chain without prematurely resolving them +- **Input**: updated graph + proposal snapshot +- **Output**: parent/ancestor status and confidence updates, additional diagnostics +- **Why it exists**: decomposition requires deterministic reconstruction as well as decomposition +- **What breaks if removed**: child answers stay local and parents never become progressively better-supported + +### 16. Confidence / completeness / corroboration update + +- **Purpose**: derive parent-level `confidenceAssessment` from resolved children and branch interactions +- **Input**: parent child set + branch evidence/status interactions +- **Output**: `evidenceConfidence`, `completenessStatus`, `conclusionConfidence`, plus derived display `confidence` +- **Why it exists**: reasoning support must be separated from completion and contradiction state +- **What breaks if removed**: parent confidence collapses back into vague status-driven heuristics + +### 17. Next active unknown + question formulation + +- **Purpose**: reselect the next unresolved unknown and formulate a concrete next question +- **Input**: updated graph + selection state + graph context +- **Output**: next active unknown and question object +- **Why it exists**: closes the recursive interaction loop +- **What breaks if removed**: the system updates the graph but cannot continue investigation deterministically + +## Architectural invariants + +The current implementation enforces these invariants: + +1. **A question must originate from an explicit unresolved unknown node.** +2. **Unknown selection is deterministic.** +3. **Alphabetical ordering is not treated as reasoning.** +4. **Relationship reasoning does not precede comparability.** +5. **The LLM never mutates the graph directly; it only proposes updates.** +6. **All graph updates are schema-validated before application.** +7. **All node/edge references must resolve to existing nodes.** +8. **Duplicate node IDs are rejected.** +9. **Duplicate added edge IDs are rejected.** +10. **A selected question cannot target a resolved unknown.** +11. **A resolved unknown updated to `resolved` must also appear in `resolvedUnknownNodeIds`.** +12. **A proposal must contain a meaningful change.** +13. **Every newly added unknown must include why-it-matters language.** +14. **Every newly added unknown must be explicitly connected to answer-derived graph structure.** +15. **Composite selected unknowns are decomposed before direct questioning when atomicity rules require it.** +16. **Parent unknowns remain unresolved until completion rules are satisfied.** +17. **Confidence must not outrun completeness.** +18. **Duplicate evidence cannot increase confidence.** +19. **Conflicting evidence caps conclusion confidence.** +20. **Cross-branch corroboration only counts for distinct branches with distinct evidence keys.** +21. **Ambiguous leading unknowns remain explicit ambiguity, not silent forced choice.** + +## Recursive loops and stopping rules + +### Main investigation loop + +```text +Unknown +↓ +Question +↓ +Answer +↓ +Proposal +↓ +Graph update +↓ +Propagation +↓ +Next unknown +``` + +- **Exit condition**: no unresolved candidates remain, or no next question is justified, or proposal/application fails +- **Stopping rule**: deterministic selection returns `null` or explicit ambiguity, or update validation blocks progress +- **Completion behaviour**: continues only while the graph contains justified unresolved unknowns + +### Decomposition loop + +```text +Selected unknown +↓ +Atomicity assessment +↓ +If composite: decompose +↓ +Reselect child +↓ +Atomicity assessment again +``` + +- **Exit condition**: selected child is atomic; parent already has children; max decomposition depth reached; or decomposition quality fails +- **Stopping rule**: `MAX_DECOMPOSITION_DEPTH`, reuse instead of regeneration, or inability to produce enough valid child unknowns +- **Completion behaviour**: deterministic and bounded; no infinite recursive decomposition path is intentionally allowed + +### Propagation loop + +```text +Resolved child +↓ +Ancestor chain walk +↓ +Recompute parent state +↓ +Stop when no ancestor state changes +``` + +- **Exit condition**: no more parents in the ancestor chain or no state change +- **Stopping rule**: ancestor chain is explicit and finite; propagation does not invent new ancestors +- **Completion behaviour**: deterministic upward traversal with explicit stop on unchanged state + +### Potential infinite loops reviewed + +- **Unknown/question recursion**: bounded by unresolved unknown set, proposal validation, and explicit no-candidate states +- **Decomposition recursion**: bounded by max depth and child reuse rules +- **Propagation recursion**: bounded by finite ancestor chain and no-change stop condition + +No intentional infinite reasoning loop is present in the implemented architecture. + +## Graph lifecycle summary + +### Node lifecycle + +1. node created by `buildInitialGraph` or later proposal/decomposition/emergent-unknown logic +2. node validated by schema +3. node may become active unknown +4. node may be updated by proposal application +5. unknown node may become `resolved`, `provisional`, `contradicted`, or remain `unknown` +6. resolved unknown ID is tracked in `resolvedNodeIds` + +### Edge lifecycle + +1. edge created in initial graph or by deterministic proposal augmentation +2. edge validated against existing node IDs +3. edge may be removed only through explicit `removedEdgeIds` +4. edge relationships also update `dependsOn` / `childIds` projections during application + +### Unknown lifecycle + +1. initial unknown discovered from reconstruction +2. selected deterministically or left ambiguous +3. may be decomposed if composite +4. may be resolved directly by answer +5. may cause emergent reasoning unknown creation when relationship reasoning demands a new explicit question target + +### Resolved lifecycle + +1. proposal marks unresolved unknown resolved +2. reconciliation ensures resolution semantics are explicit +3. `resolvedUnknownNodeIds` feed graph application +4. propagation may resolve parent only when completion rule is met + +### Confidence lifecycle + +1. nodes begin with base `confidence` +2. parent/ancestor propagation derives `confidenceAssessment` +3. display `confidence` is derived from `conclusionConfidence` +4. completeness, duplicate evidence, contradiction, and corroboration constrain the result + +### Question lifecycle + +1. selected unknown becomes question target +2. `formulateQuestion` or tie-resolution logic produces question text +3. answer returns through update route +4. proposal may select a new question target or leave reselection to deterministic logic + +Every major transition above is explicit in the current codebase rather than implicit in model text alone. + +## Duplicated or overlapping concepts + +The following concepts are intentionally close and may look duplicated: + +- **status vs confidence**: status captures lifecycle/progression; confidence captures support strength +- **confidence vs confidenceAssessment**: `confidence` is now a derived display field, while `confidenceAssessment` carries separated reasoning dimensions +- **resolvedNodeIds vs node.status === resolved**: both are maintained; the first is a graph-level index, the second is node-local state +- **selectedQuestion in proposal vs selectedQuestion in final result**: proposal may omit or propose one, final result recomputes deterministic selection/questioning after graph logic +- **comparability state in reasoningState vs derived comparability from graph**: overrides may carry forward prior confirmed reasoning, but `buildReasoningState` still rebuilds from graph + override context + +These are not necessarily defects, but they are the main places where future simplification pressure is likely. + +## Known boundaries and deliberate exclusions + +v0.6 deliberately does **not** attempt the following: + +- probabilistic reasoning +- Bayesian inference +- persistence +- semantic embeddings +- fuzzy semantic similarity +- autonomous exploration outside explicit user answers +- multi-hop corroboration across unrelated subtrees without a shared direct parent +- expert-only jargon-specific reasoning modes +- UI-heavy reasoning visualisation beyond existing graph/update displays +- arbitrary non-deterministic tie breaking + +## Defects found during this review + +No new production defect was intentionally introduced or fixed as part of this architecture review. + +## Developer notes + +- `startCase` owns reconstruction → graph build → first deterministic selection. +- `updateCaseWithDependencies` owns proposal generation / parsing and delegates deterministic graph semantics to `applyValidatedProposal`. +- `applyValidatedProposal` is the main reasoning pipeline coordinator for update-time graph evolution. +- `question-formulator.js` owns comparability, relationship classification, atomicity assessment, investigation strategy selection, and question formulation. +- `utils.js` owns selection scoring, ordering, graph validation, and safe graph update application. From 5049435005c18bfa24c10fa5f6752e230a1637db Mon Sep 17 00:00:00 2001 From: robbond Date: Mon, 3 Aug 2026 07:38:05 +0100 Subject: [PATCH 17/17] docs: add v0.6 release notes --- docs/v0.6-release-notes.md | 89 +++++++++++++++++++ .../selection-influence-diagnostic.test.js | 2 +- 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 docs/v0.6-release-notes.md diff --git a/docs/v0.6-release-notes.md b/docs/v0.6-release-notes.md new file mode 100644 index 0000000..e627b6a --- /dev/null +++ b/docs/v0.6-release-notes.md @@ -0,0 +1,89 @@ +# v0.6 Release Notes + +## Purpose + +v0.6 turns the engine into a deterministic recursive reasoning system that keeps next questions, decomposition, propagation, and confidence updates explicitly grounded in the situation graph. + +## Capabilities added + +- deterministic unknown selection explanations +- explicit ambiguity handling instead of silent tie-breaking +- comparability assessment before relationship reasoning +- relationship classification after comparability +- reasoning-stage progression after comparability answers +- graph-backed next questions via explicit unknown nodes +- investigation-strategy-based question formulation +- atomicity assessment for selected unknowns +- composite-unknown decomposition into child unknowns +- child-quality validation for decomposition outputs +- upward propagation from resolved children to parents and ancestors +- separation of evidence confidence, completeness, and conclusion confidence +- deterministic cross-branch corroboration, conflict, and duplicate-evidence handling +- developer-facing reasoning architecture documentation + +## Reasoning pipeline summary + +```text +Scenario +→ Reconstruction +→ Initial graph +→ Deterministic unknown selection +→ Question +→ Answer +→ Proposal +→ Proposal parsing / validation +→ Graph update +→ Reasoning-state rebuild +→ Comparability assessment +→ Relationship classification +→ Emergent unknown creation / reuse +→ Atomicity assessment +→ Optional decomposition +→ Propagation +→ Confidence / completeness / corroboration update +→ Next active unknown +→ Next question +``` + +## Core invariants + +- every asked question must originate from an explicit unresolved unknown +- unknown selection is deterministic +- ambiguity is preserved explicitly when no justified distinction exists +- relationship reasoning cannot precede comparability +- parent nodes cannot resolve before completion rules are met +- confidence cannot outrun completeness +- duplicate evidence cannot increase confidence +- conflicting evidence caps conclusion confidence +- cross-branch corroboration only counts for distinct branches with distinct evidence keys +- the LLM proposes updates but does not mutate the graph directly + +## What v0.6 proved + +- graph-backed questioning works better when every justified next question maps to an explicit unresolved node +- broad unknowns can be decomposed deterministically before direct questioning +- resolved child evidence can be propagated upward without prematurely resolving parent reasoning +- confidence becomes easier to reason about when evidence quality, completeness, and conclusion strength are separated +- deterministic cross-branch corroboration can improve support without double-counting repeated evidence + +## Known limitations + +- sibling selection still depends on the existing deterministic scorer and may choose a justified next branch that is not always the intuitively expected one +- cross-branch corroboration is limited to direct child branches of the same parent +- no multi-hop corroboration exists across unrelated subtrees +- reasoning remains bounded to explicitly represented graph structure and user-provided answers + +## Deliberate exclusions + +- no persistence +- no autonomous exploration +- no probabilistic reasoning +- no Bayesian reasoning +- no semantic embeddings +- no expert mode +- no multi-hop corroboration across unrelated subtrees +- no heavy graph visualisation + +## Next experimental question + +`Can the engine preserve and reuse successful reasoning structures across separate cases without turning prior experience into unquestioned assumptions?` diff --git a/tests/graph/selection-influence-diagnostic.test.js b/tests/graph/selection-influence-diagnostic.test.js index a38521b..8bd0d3b 100644 --- a/tests/graph/selection-influence-diagnostic.test.js +++ b/tests/graph/selection-influence-diagnostic.test.js @@ -290,7 +290,7 @@ describe("selection influence diagnostic", () => { expect(diagnosticRecord.neutralSelection.status).toBe("ambiguous"); expect(diagnosticRecord.selectedExplanationContributions).toBeUndefined(); expect(diagnosticRecord.tieQuestion).toBe( - "What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?", + "Were these figures measured on the same basis and at the same scale?", ); expect(diagnosticRecord.tieQuestion.toLowerCase()).not.toMatch( /accounts receivable|capex|debt repayments|working capital/,