From b00928d6fbefa7581186df685d76495850f854e1 Mon Sep 17 00:00:00 2001 From: robbond Date: Mon, 3 Aug 2026 10:25:26 +0100 Subject: [PATCH] feat: introduce reasoning pattern selection --- docs/v0.7-question-simplicity-experiment.md | 89 +++ lib/graph/apply-proposal.js | 18 + lib/graph/orchestrator.js | 70 +++ lib/graph/question-formulator.js | 557 +++++++++++++++++- tests/graph/question-formulator.test.js | 65 ++ tests/graph/question-simplicity.test.js | 5 + .../graph/reasoning-pattern-selection.test.js | 201 +++++++ 7 files changed, 986 insertions(+), 19 deletions(-) create mode 100644 tests/graph/reasoning-pattern-selection.test.js diff --git a/docs/v0.7-question-simplicity-experiment.md b/docs/v0.7-question-simplicity-experiment.md index dcf2e40..bdeca1b 100644 --- a/docs/v0.7-question-simplicity-experiment.md +++ b/docs/v0.7-question-simplicity-experiment.md @@ -94,6 +94,95 @@ The engine now decomposes whenever either of these is true: This prevents a broad container node from becoming the selected question target even when its wording looks grammatically singular. +## Reasoning Pattern + +The next failure exposed a deeper issue: even after atomicity and answerability were added, the engine could still choose a question template from the wrong reasoning family. + +The live failure was an explanation-style prompt appearing in a commercial validation scenario: + +> What changed during the period that could help explain why ... + +That was wrong not because of wording, but because the engine had selected an **explanation family** when the actual task was a **decision investigation**. + +To correct that, the deterministic pipeline now explicitly inserts a reasoning-pattern stage: + +```text +selected unknown +→ atomicity +→ answerability +→ reasoning pattern +→ investigation strategy +→ question family +→ question +``` + +This matters because each stage must constrain the next. + +- **Reasoning Pattern** decides what kind of reasoning is happening +- **Investigation Strategy** decides how to reduce uncertainty within that pattern +- **Question Family** decides what template space is allowed +- **Question** is the final concrete wording + +Without this stage separation, strategy and template selection can leak across domains and reuse relationship/explanation prompts too broadly. + +## Deterministic reasoning-pattern vocabulary + +The current deterministic pattern vocabulary is intentionally small: + +- decision +- explanation +- contradiction +- definition +- diagnosis +- comparison +- prioritisation + +Pattern selection uses graph structure rather than wording alone, including: + +- node kind +- relationship / observation topology +- parent context +- reasoning state +- selected unknown role in the graph + +## Question-family mapping + +Patterns now constrain which question families are allowed. + +- **decision** + - decision_foundation + - decision_evidence + - decision_threshold + - definition +- **explanation** + - explanation + - comparison +- **contradiction** + - contradiction + - comparison + - explanation +- **definition** + - definition +- **diagnosis** + - diagnosis + - comparison +- **comparison** + - comparison +- **prioritisation** + - prioritisation + - decision_threshold + +Most importantly: + +- explanation templates are only allowed for `explanation` or `contradiction` +- decision investigations cannot emit explanation-family questions + +## Live correction + +For the commercial-method scenario, the engine now classifies the reasoning as a **decision** pattern rather than an explanation pattern. + +That means explanation-family templates are explicitly rejected, and the selected child unknown must be questioned using a decision-compatible family instead. + ## UI result The long compound question no longer survives as the first follow-up in the tested path. diff --git a/lib/graph/apply-proposal.js b/lib/graph/apply-proposal.js index d644aad..88a995e 100644 --- a/lib/graph/apply-proposal.js +++ b/lib/graph/apply-proposal.js @@ -1766,6 +1766,15 @@ export function determineGraphBackedQuestion({ situationGraph }) { reason: formulatedQuestion.reason, strategy: formulatedQuestion.strategy, investigationStrategy: formulatedQuestion.investigationStrategy, + reasoningPattern: formulatedQuestion.reasoningPattern, + reasoningPatternReason: formulatedQuestion.reasoningPatternReason, + questionFamily: formulatedQuestion.questionFamily, + allowedQuestionFamilies: + formulatedQuestion.allowedQuestionFamilies, + rejectedQuestionFamilies: + formulatedQuestion.rejectedQuestionFamilies, + selectedQuestionTemplate: + formulatedQuestion.selectedQuestionTemplate, questionComplexity: formulatedQuestion.questionComplexity, plainLanguageNormalisations: formulatedQuestion.plainLanguageNormalisations, @@ -2469,6 +2478,15 @@ export function applyValidatedProposal({ reason: formulatedQuestion?.reason || deterministicSelection.reason, strategy: formulatedQuestion?.strategy, investigationStrategy: formulatedQuestion?.investigationStrategy, + reasoningPattern: formulatedQuestion?.reasoningPattern, + reasoningPatternReason: formulatedQuestion?.reasoningPatternReason, + questionFamily: formulatedQuestion?.questionFamily, + allowedQuestionFamilies: + formulatedQuestion?.allowedQuestionFamilies, + rejectedQuestionFamilies: + formulatedQuestion?.rejectedQuestionFamilies, + selectedQuestionTemplate: + formulatedQuestion?.selectedQuestionTemplate, questionComplexity, plainLanguageNormalisations, } diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index a622a62..7a34ff9 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -59,6 +59,12 @@ function buildDiagnostics({ decompositionReason, selectedContainerUnknown, selectedChildUnknown, + reasoningPattern, + questionFamily, + allowedQuestionFamilies, + rejectedQuestionFamilies, + selectedQuestionTemplate, + reasoningPatternReason, }) { return { promptVersion: analysis?.promptVersion ?? null, @@ -88,6 +94,12 @@ function buildDiagnostics({ decompositionReason: decompositionReason ?? null, selectedContainerUnknown: selectedContainerUnknown ?? null, selectedChildUnknown: selectedChildUnknown ?? null, + reasoningPattern: reasoningPattern ?? null, + questionFamily: questionFamily ?? null, + allowedQuestionFamilies: allowedQuestionFamilies ?? [], + rejectedQuestionFamilies: rejectedQuestionFamilies ?? [], + selectedQuestionTemplate: selectedQuestionTemplate ?? null, + reasoningPatternReason: reasoningPatternReason ?? null, }; } @@ -177,6 +189,12 @@ function buildUpdateDiagnostics({ selectedUnknownBefore, selectedUnknownAfter, plainLanguageNormalisations, + reasoningPattern, + questionFamily, + allowedQuestionFamilies, + rejectedQuestionFamilies, + selectedQuestionTemplate, + reasoningPatternReason, }) { return { promptVersion: promptVersion ?? "v0.4", @@ -257,6 +275,12 @@ function buildUpdateDiagnostics({ selectedUnknownBefore: selectedUnknownBefore ?? null, selectedUnknownAfter: selectedUnknownAfter ?? null, plainLanguageNormalisations: plainLanguageNormalisations ?? [], + reasoningPattern: reasoningPattern ?? null, + questionFamily: questionFamily ?? null, + allowedQuestionFamilies: allowedQuestionFamilies ?? [], + rejectedQuestionFamilies: rejectedQuestionFamilies ?? [], + selectedQuestionTemplate: selectedQuestionTemplate ?? null, + reasoningPatternReason: reasoningPatternReason ?? null, unknownSelectionExplanation: unknownSelectionExplanation ?? null, }; } @@ -370,6 +394,21 @@ export async function startCase(body) { initialQuestionResult.selectedContainerUnknown ?? null, selectedChildUnknown: initialQuestionResult.selectedChildUnknown ?? null, + reasoningPattern: + initialQuestionResult.selectedQuestion?.reasoningPattern ?? null, + questionFamily: + initialQuestionResult.selectedQuestion?.questionFamily ?? null, + allowedQuestionFamilies: + initialQuestionResult.selectedQuestion?.allowedQuestionFamilies ?? [], + rejectedQuestionFamilies: + initialQuestionResult.selectedQuestion?.rejectedQuestionFamilies ?? + [], + selectedQuestionTemplate: + initialQuestionResult.selectedQuestion?.selectedQuestionTemplate ?? + null, + reasoningPatternReason: + initialQuestionResult.selectedQuestion?.reasoningPatternReason ?? + null, }), validationErrors: graphReferenceValidation.errors, statusCode: 500, @@ -413,6 +452,19 @@ export async function startCase(body) { selectedContainerUnknown: initialQuestionResult.selectedContainerUnknown ?? null, selectedChildUnknown: initialQuestionResult.selectedChildUnknown ?? null, + reasoningPattern: + initialQuestionResult.selectedQuestion?.reasoningPattern ?? null, + questionFamily: + initialQuestionResult.selectedQuestion?.questionFamily ?? null, + allowedQuestionFamilies: + initialQuestionResult.selectedQuestion?.allowedQuestionFamilies ?? [], + rejectedQuestionFamilies: + initialQuestionResult.selectedQuestion?.rejectedQuestionFamilies ?? [], + selectedQuestionTemplate: + initialQuestionResult.selectedQuestion?.selectedQuestionTemplate ?? + null, + reasoningPatternReason: + initialQuestionResult.selectedQuestion?.reasoningPatternReason ?? null, }), }; } @@ -709,6 +761,18 @@ async function updateCaseWithDependencies(body, dependencies = {}) { selectedUnknownAfter: applicationResult.selectedUnknownAfter, plainLanguageNormalisations: applicationResult.plainLanguageNormalisations, + reasoningPattern: + applicationResult.selectedQuestion?.reasoningPattern ?? null, + questionFamily: + applicationResult.selectedQuestion?.questionFamily ?? null, + allowedQuestionFamilies: + applicationResult.selectedQuestion?.allowedQuestionFamilies ?? [], + rejectedQuestionFamilies: + applicationResult.selectedQuestion?.rejectedQuestionFamilies ?? [], + selectedQuestionTemplate: + applicationResult.selectedQuestion?.selectedQuestionTemplate ?? null, + reasoningPatternReason: + applicationResult.selectedQuestion?.reasoningPatternReason ?? null, unknownSelectionExplanation: buildUnknownSelectionDiagnostics( applicationResult.updatedSituationGraph, applicationResult.updatedSituationGraph.resolvedNodeIds || [], @@ -787,6 +851,12 @@ async function updateCaseWithDependencies(body, dependencies = {}) { selectedUnknownBefore: null, selectedUnknownAfter: null, plainLanguageNormalisations: [], + reasoningPattern: null, + questionFamily: null, + allowedQuestionFamilies: [], + rejectedQuestionFamilies: [], + selectedQuestionTemplate: null, + reasoningPatternReason: null, unknownSelectionExplanation: buildUnknownSelectionDiagnostics( situationGraph, situationGraph.resolvedNodeIds || [], diff --git a/lib/graph/question-formulator.js b/lib/graph/question-formulator.js index cb556f2..111f7f8 100644 --- a/lib/graph/question-formulator.js +++ b/lib/graph/question-formulator.js @@ -527,6 +527,10 @@ function buildFoundationalDirectQuestion(node) { function isRelationshipExplanationUnknown(node, graph) { const text = normaliseText(`${node?.label || ""} ${node?.description || ""}`); + if (isDirectlyAnswerableObservationChildText(text)) { + return false; + } + return ( collectObservationNodes(graph).length >= 2 && /\b(explain|explanation|divergence|moved differently|difference between|change or event|what changed|why the observations)/.test( @@ -729,6 +733,433 @@ export function assessUnknownAtomicity({ node, graph }) { }; } +const ALL_REASONING_PATTERNS = [ + "decision", + "explanation", + "contradiction", + "definition", + "diagnosis", + "comparison", + "prioritisation", +]; + +const QUESTION_FAMILIES_BY_PATTERN = { + decision: [ + "decision_foundation", + "decision_evidence", + "decision_threshold", + "definition", + ], + explanation: ["explanation", "comparison"], + contradiction: ["contradiction", "comparison", "explanation"], + definition: ["definition"], + diagnosis: ["diagnosis", "comparison"], + comparison: ["comparison"], + prioritisation: ["prioritisation", "decision_threshold"], +}; + +const STRATEGIES_BY_PATTERN = { + decision: [ + "decision_threshold", + "evidence_gathering", + "definition", + "baseline_reconstruction", + ], + explanation: ["evidence_gathering", "baseline_reconstruction"], + contradiction: [ + "contradiction_resolution", + "baseline_reconstruction", + "evidence_gathering", + ], + definition: ["definition"], + diagnosis: ["evidence_gathering", "baseline_reconstruction"], + comparison: ["baseline_reconstruction", "evidence_gathering"], + prioritisation: ["decision_threshold", "evidence_gathering"], +}; + +function buildParentChain(node, graph) { + const nodesById = buildNodeMap(graph); + const chain = []; + let current = node?.parentId ? nodesById.get(node.parentId) : null; + + while (current) { + chain.push(current); + current = current.parentId ? nodesById.get(current.parentId) : null; + } + + return chain; +} + +function hasDecisionContext(node, graph, relatedNodes = []) { + const ancestry = buildParentChain(node, graph); + const contextText = normaliseText( + [ + graph?.centralStatement, + node?.label, + node?.description, + ...relatedNodes.map((relatedNode) => relatedNode.label), + ...relatedNodes.map((relatedNode) => relatedNode.description), + ...ancestry.map((ancestor) => ancestor.label), + ...ancestry.map((ancestor) => ancestor.description), + ...collectResolvedContextValues(graph), + ] + .filter(Boolean) + .join(" "), + ); + + return /\b(whether to|build|launch|continue|proceed|invest|commercially justified|commercial justification|commercial value|business case|viability)\b/.test( + contextText, + ); +} + +function hasObservationRelationshipTopology(node, graph, relatedNodes = []) { + const ancestry = buildParentChain(node, graph); + const topologyNodes = [node, ...relatedNodes, ...ancestry].filter(Boolean); + + return ( + collectObservationNodes(graph).length >= 2 && + topologyNodes.some( + (candidate) => + candidate.kind === "observation" || candidate.kind === "relationship", + ) + ); +} + +function isDefinitionPatternCandidate(node, graph) { + const text = normaliseText(`${node?.label || ""} ${node?.description || ""}`); + return isDefinitionLikeUnknown( + text, + `${text} ${graph?.centralStatement || ""}`, + ); +} + +function isContradictionPatternCandidate(node, graph, relatedNodes = []) { + const relationshipStatus = graph?.reasoningState?.relationshipStatus ?? null; + const text = normaliseText( + [ + node?.label, + node?.description, + ...relatedNodes.map((relatedNode) => relatedNode.label), + ...relatedNodes.map((relatedNode) => relatedNode.description), + ] + .filter(Boolean) + .join(" "), + ); + + return ( + relationshipStatus === "contradictory" || + /\b(contradiction|contradict|conflict|inconsistent|mismatch|opposing)\b/.test( + text, + ) + ); +} + +function isComparisonPatternCandidate(node, graph, relatedNodes = []) { + if (isRelationshipExplanationUnknown(node, graph)) { + return false; + } + + const text = normaliseText( + [ + node?.label, + node?.description, + ...relatedNodes.map((relatedNode) => relatedNode.label), + ...relatedNodes.map((relatedNode) => relatedNode.description), + ] + .filter(Boolean) + .join(" "), + ); + + return ( + (hasObservationRelationshipTopology(node, graph, relatedNodes) || + collectObservationNodes(graph).length >= 2 || + graph?.reasoningState?.comparabilityStatus === "uncertain") && + /\b(compare|comparison|different timing|measured|measurement|basis|scale|period|alternative|alternatives|better than)\b/.test( + text, + ) + ); +} + +function isExplanationPatternCandidate(node, graph, relatedNodes = []) { + if (isRelationshipExplanationUnknown(node, graph)) return true; + + const text = normaliseText( + [ + node?.label, + node?.description, + ...relatedNodes.map((relatedNode) => relatedNode.label), + ...relatedNodes.map((relatedNode) => relatedNode.description), + ] + .filter(Boolean) + .join(" "), + ); + + return ( + hasObservationRelationshipTopology(node, graph, relatedNodes) && + /\b(explain|explanation|why .* but|difference between|divergence|what changed|moved differently)\b/.test( + text, + ) + ); +} + +function isPrioritisationPatternCandidate(node, graph, relatedNodes = []) { + const text = normaliseText( + [ + node?.label, + node?.description, + graph?.centralStatement, + ...relatedNodes.map((relatedNode) => relatedNode.label), + ] + .filter(Boolean) + .join(" "), + ); + + return /\b(prioritise|prioritize|priority|rank|ranking|trade off|tradeoff|which first)\b/.test( + text, + ); +} + +export function selectReasoningPattern({ node, graph, context = {} }) { + const relatedNodes = collectRelatedNodes(node, graph); + const patternContext = { + hasDecisionContext: hasDecisionContext(node, graph, relatedNodes), + hasObservationRelationshipTopology: hasObservationRelationshipTopology( + node, + graph, + relatedNodes, + ), + }; + + if (isDefinitionPatternCandidate(node, graph)) { + return { + pattern: "definition", + reason: + "Selected definition because the active unknown is about meaning, scope, or term boundaries.", + context: patternContext, + }; + } + + if (isContradictionPatternCandidate(node, graph, relatedNodes)) { + return { + pattern: "contradiction", + reason: + "Selected contradiction because the graph indicates opposing claims or incompatible observations.", + context: patternContext, + }; + } + + if (isComparisonPatternCandidate(node, graph, relatedNodes)) { + return { + pattern: "comparison", + reason: + "Selected comparison because the active unknown is about distinguishing measurements, timing, basis, or alternatives.", + context: patternContext, + }; + } + + if (isExplanationPatternCandidate(node, graph, relatedNodes)) { + return { + pattern: "explanation", + reason: + "Selected explanation because the active unknown is about accounting for a relationship between observations.", + context: patternContext, + }; + } + + if (patternContext.hasDecisionContext) { + return { + pattern: "decision", + reason: + "Selected decision because the active unknown sits inside a build, continue, invest, or commercial-justification decision context.", + context: patternContext, + }; + } + + if (isPrioritisationPatternCandidate(node, graph, relatedNodes)) { + return { + pattern: "prioritisation", + reason: + "Selected prioritisation because the active unknown is about ordering options or trade-offs.", + context: patternContext, + }; + } + + return { + pattern: "diagnosis", + reason: + "Selected diagnosis as the default because the active unknown needs clarifying evidence or mechanism-level investigation.", + context: patternContext, + }; +} + +function allowedQuestionFamiliesForPattern(pattern) { + return QUESTION_FAMILIES_BY_PATTERN[pattern] || [pattern]; +} + +function rejectedQuestionFamiliesForPattern(pattern) { + const allowed = new Set(allowedQuestionFamiliesForPattern(pattern)); + return ALL_REASONING_PATTERNS.flatMap((candidatePattern) => + ( + QUESTION_FAMILIES_BY_PATTERN[candidatePattern] || [candidatePattern] + ).filter((family) => !allowed.has(family)), + ).filter((family, index, list) => list.indexOf(family) === index); +} + +function constrainStrategyToReasoningPattern(strategy, reasoningPattern) { + if (!strategy) return null; + const allowedStrategies = STRATEGIES_BY_PATTERN[reasoningPattern] || []; + return allowedStrategies.includes(strategy.key) ? strategy : null; +} + +function selectQuestionFamily({ + node, + graph, + reasoningPattern, + investigationStrategy, +}) { + const text = normaliseText(`${node?.label || ""} ${node?.description || ""}`); + + if (reasoningPattern === "definition") { + return { family: "definition", template: "definition_meaning" }; + } + + if (reasoningPattern === "contradiction") { + if (/\b(period|timing|basis|scale|measure|measured)\b/.test(text)) { + return { + family: "comparison", + template: "comparison_reconcile_measurement", + }; + } + return { + family: "contradiction", + template: "contradiction_resolve_opposition", + }; + } + + if (reasoningPattern === "explanation") { + return { + family: "explanation", + template: "explanation_broad_investigation", + }; + } + + if (reasoningPattern === "comparison") { + return /\b(period|timing)\b/.test(text) && + !/\bhow the two observations were measured|measurement\b/.test(text) + ? { family: "comparison", template: "comparison_timing_basis" } + : { family: "comparison", template: "comparison_measurement_basis" }; + } + + if (reasoningPattern === "decision") { + if ( + /\b(audience|customer|user|buyer|stakeholder|recipient|who experiences)\b/.test( + text, + ) + ) { + return { family: "decision_foundation", template: "decision_audience" }; + } + if ( + /\b(alternative|alternatives|better than|different from|deal with)\b/.test( + text, + ) + ) { + return { + family: "decision_foundation", + template: "decision_current_alternatives", + }; + } + if (/\b(problem|need|demand)\b/.test(text)) { + return { + family: "decision_foundation", + template: "decision_problem_existence", + }; + } + if (investigationStrategy?.key === "decision_threshold") { + return { + family: "decision_threshold", + template: "decision_threshold_outcome", + }; + } + return { + family: "decision_evidence", + template: "decision_evidence_clarification", + }; + } + + if (reasoningPattern === "prioritisation") { + return { family: "prioritisation", template: "prioritisation_tradeoff" }; + } + + return investigationStrategy?.key === "baseline_reconstruction" + ? { family: "comparison", template: "diagnosis_baseline_comparison" } + : { family: "diagnosis", template: "diagnosis_evidence" }; +} + +function buildQuestionFromFamily({ + node, + graph, + reasoningPattern, + questionFamily, + selectedQuestionTemplate, + investigationStrategy, +}) { + const meaning = extractMeaning(node); + + if (reasoningPattern === "decision") { + if (selectedQuestionTemplate === "decision_audience") { + return "Who experiences this problem?"; + } + if (selectedQuestionTemplate === "decision_current_alternatives") { + return "How do people deal with this today?"; + } + if (selectedQuestionTemplate === "decision_problem_existence") { + return "What makes you think this is a real problem?"; + } + if (selectedQuestionTemplate === "decision_threshold_outcome") { + return buildQuestionFromStrategy( + investigationStrategy || { + key: "decision_threshold", + meaning, + actionPhrase: null, + }, + ); + } + return `What evidence would clarify ${stripTrailingPunctuation(meaning)}?`; + } + + if (questionFamily === "definition") { + return `What does ${meaning} mean in this situation?`; + } + + if (reasoningPattern === "comparison") { + if (selectedQuestionTemplate === "comparison_timing_basis") { + return `What evidence would clarify whether ${stripTrailingPunctuation(meaning)}?`; + } + if (selectedQuestionTemplate === "comparison_measurement_basis") { + return `What evidence would clarify ${stripTrailingPunctuation(meaning)}?`; + } + return `What evidence would clarify ${stripTrailingPunctuation(meaning)}?`; + } + + if (reasoningPattern === "contradiction") { + return investigationStrategy?.key === "contradiction_resolution" + ? buildQuestionFromStrategy(investigationStrategy) + : `What fact would resolve the contradiction about ${stripTrailingPunctuation(meaning)}?`; + } + + if (reasoningPattern === "explanation") { + return buildBroadInvestigationQuestion(graph); + } + + if (reasoningPattern === "prioritisation") { + return `Which option should be investigated first, and why?`; + } + + return investigationStrategy + ? buildQuestionFromStrategy(investigationStrategy) + : buildNeutralClarificationQuestion(meaning); +} + export function formulateTieResolutionQuestion({ graph }) { const comparability = assessComparability(graph); if (comparability.comparabilityStatus === "uncertain") { @@ -747,6 +1178,14 @@ export function formulateTieResolutionQuestion({ graph }) { relationshipStatus: deferredRelationship.relationshipStatus, relationshipReason: deferredRelationship.reason, relationshipAssessed: deferredRelationship.relationshipAssessed, + reasoningPattern: "comparison", + reasoningPatternReason: + "Tie resolution is using the comparison family because comparability is still unresolved.", + allowedQuestionFamilies: allowedQuestionFamiliesForPattern("comparison"), + rejectedQuestionFamilies: + rejectedQuestionFamiliesForPattern("comparison"), + questionFamily: "comparison", + selectedQuestionTemplate: "comparison_tie_resolution", questionRequired: true, reasoningStages: deferredRelationship.reasoningStages, }; @@ -766,6 +1205,14 @@ export function formulateTieResolutionQuestion({ graph }) { relationshipReason: relationship.reason, relationshipAssessed: relationship.relationshipAssessed, contradictionReasoningAllowed: relationship.contradictionReasoningAllowed, + reasoningPattern: "comparison", + reasoningPatternReason: + "Tie resolution remains in the comparison family because no distinct winning unknown exists.", + allowedQuestionFamilies: allowedQuestionFamiliesForPattern("comparison"), + rejectedQuestionFamilies: + rejectedQuestionFamiliesForPattern("comparison"), + questionFamily: "comparison", + selectedQuestionTemplate: "comparison_no_question_required", questionRequired: relationship.questionRequired, questionSuppressedReason: relationship.questionSuppressedReason, reasoningStages: relationship.reasoningStages, @@ -786,6 +1233,14 @@ export function formulateTieResolutionQuestion({ graph }) { relationshipReason: relationship.reason, relationshipAssessed: relationship.relationshipAssessed, contradictionReasoningAllowed: relationship.contradictionReasoningAllowed, + reasoningPattern: "explanation", + reasoningPatternReason: + "Tie resolution is using the explanation family because the observations appear related and need a neutral explanation question.", + allowedQuestionFamilies: allowedQuestionFamiliesForPattern("explanation"), + rejectedQuestionFamilies: + rejectedQuestionFamiliesForPattern("explanation"), + questionFamily: "explanation", + selectedQuestionTemplate: "explanation_broad_investigation", questionRequired: relationship.questionRequired, reasoningStages: relationship.reasoningStages, }; @@ -812,6 +1267,14 @@ export function formulateTieResolutionQuestion({ graph }) { relationshipReason: relationship.reason, relationshipAssessed: relationship.relationshipAssessed, contradictionReasoningAllowed: relationship.contradictionReasoningAllowed, + reasoningPattern: "contradiction", + reasoningPatternReason: + "Tie resolution is using the contradiction family because the graph is trying to distinguish incompatible explanations.", + allowedQuestionFamilies: allowedQuestionFamiliesForPattern("contradiction"), + rejectedQuestionFamilies: + rejectedQuestionFamiliesForPattern("contradiction"), + questionFamily: "contradiction", + selectedQuestionTemplate: "contradiction_distinguishing_change", questionRequired: relationship.questionRequired, reasoningStages: relationship.reasoningStages, }; @@ -899,6 +1362,9 @@ function buildInvestigationStrategy({ } export function selectInvestigationStrategy({ node, graph, context = {} }) { + const effectiveReasoningPattern = + context.reasoningPattern || + selectReasoningPattern({ node, graph, context }).pattern; const relatedNodes = collectRelatedNodes(node, graph); const meaning = extractMeaning(node); const combinedText = [ @@ -949,8 +1415,10 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) { const hasPrimaryBaselineLanguage = /\b(before|previous|baseline|prior|comparable state)\b/.test(nodeText); + let selectedStrategy = null; + if (hasBaselineLanguage && hasPrimaryBaselineLanguage) { - return buildInvestigationStrategy({ + selectedStrategy = buildInvestigationStrategy({ key: "baseline_reconstruction", reason: "Selected because the unknown explicitly references a missing previous or baseline state.", @@ -991,8 +1459,11 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) { relatedNode.kind === "conclusion", ); - if (hasPrimaryDefinitionLanguage || hasDefinitionLanguage) { - return buildInvestigationStrategy({ + if ( + !selectedStrategy && + (hasPrimaryDefinitionLanguage || hasDefinitionLanguage) + ) { + selectedStrategy = buildInvestigationStrategy({ key: "definition", reason: "Selected because the unknown is primarily about clarifying what a term means in this case.", @@ -1004,8 +1475,8 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) { }); } - if (hasDecisionValueLanguage || hasCriteriaLanguage) { - return buildInvestigationStrategy({ + if (!selectedStrategy && (hasDecisionValueLanguage || hasCriteriaLanguage)) { + selectedStrategy = buildInvestigationStrategy({ key: "decision_threshold", reason: "Selected because the unknown determines the threshold for making or justifying a decision.", @@ -1017,8 +1488,11 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) { }); } - if (hasPrimaryBaselineLanguage || hasBaselineLanguage) { - return buildInvestigationStrategy({ + if ( + !selectedStrategy && + (hasPrimaryBaselineLanguage || hasBaselineLanguage) + ) { + selectedStrategy = buildInvestigationStrategy({ key: "baseline_reconstruction", reason: "Selected because reconstructing the prior state is the most direct way to resolve the unknown.", @@ -1030,8 +1504,8 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) { }); } - if (hasContradictionLanguage) { - return buildInvestigationStrategy({ + if (!selectedStrategy && hasContradictionLanguage) { + selectedStrategy = buildInvestigationStrategy({ key: "contradiction_resolution", reason: "Selected because the graph context indicates conflicting claims or inconsistent states that must be reconciled.", @@ -1043,8 +1517,11 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) { }); } - if (hasEvidenceLanguage || hasMeasurementLanguage || hasConstraintLanguage) { - return buildInvestigationStrategy({ + if ( + !selectedStrategy && + (hasEvidenceLanguage || hasMeasurementLanguage || hasConstraintLanguage) + ) { + selectedStrategy = buildInvestigationStrategy({ key: "evidence_gathering", reason: hasConstraintLanguage && hasPrimaryConstraintLanguage @@ -1058,7 +1535,10 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) { }); } - return null; + return constrainStrategyToReasoningPattern( + selectedStrategy, + effectiveReasoningPattern, + ); } function buildQuestionFromStrategy(strategy) { @@ -1232,8 +1712,22 @@ export function formulateQuestion({ node, graph, context = {} }) { return formulateTieResolutionQuestion({ graph }); } + const reasoningPatternSelection = selectReasoningPattern({ + node, + graph, + context, + }); + const allowedQuestionFamilies = allowedQuestionFamiliesForPattern( + reasoningPatternSelection.pattern, + ); + const rejectedQuestionFamilies = rejectedQuestionFamiliesForPattern( + reasoningPatternSelection.pattern, + ); const foundationalDirectQuestion = buildFoundationalDirectQuestion(node); - if (foundationalDirectQuestion) { + if ( + foundationalDirectQuestion && + reasoningPatternSelection.pattern === "decision" + ) { const plainLanguage = applyPlainLanguageNormalisations( sanitizeQuestionText(foundationalDirectQuestion), ); @@ -1249,6 +1743,12 @@ export function formulateQuestion({ node, graph, context = {} }) { "Formulated as a direct foundational question because this child unknown should be answered one step at a time.", strategy: null, investigationStrategy: null, + reasoningPattern: reasoningPatternSelection.pattern, + reasoningPatternReason: reasoningPatternSelection.reason, + questionFamily: "decision_foundation", + allowedQuestionFamilies, + rejectedQuestionFamilies, + selectedQuestionTemplate: "decision_foundation_direct_child", questionComplexity, plainLanguageNormalisations: plainLanguage.normalisations, }; @@ -1257,14 +1757,27 @@ export function formulateQuestion({ node, graph, context = {} }) { const investigationStrategy = selectInvestigationStrategy({ node, graph, - context, + context: { + ...context, + reasoningPattern: reasoningPatternSelection.pattern, + }, }); - let question = investigationStrategy - ? buildQuestionFromStrategy(investigationStrategy) - : isRelationshipExplanationUnknown(node, graph) - ? buildBroadInvestigationQuestion(graph) - : buildNeutralClarificationQuestion(extractMeaning(node)); + const questionFamilySelection = selectQuestionFamily({ + node, + graph, + reasoningPattern: reasoningPatternSelection.pattern, + investigationStrategy, + }); + + let question = buildQuestionFromFamily({ + node, + graph, + reasoningPattern: reasoningPatternSelection.pattern, + questionFamily: questionFamilySelection.family, + selectedQuestionTemplate: questionFamilySelection.template, + investigationStrategy, + }); question = sanitizeQuestionText(question); const plainLanguage = applyPlainLanguageNormalisations(question); @@ -1310,6 +1823,12 @@ export function formulateQuestion({ node, graph, context = {} }) { : "Formulated as a neutral clarification question because no narrower investigation strategy clearly applied.", strategy: investigationStrategy?.key ?? null, investigationStrategy, + reasoningPattern: reasoningPatternSelection.pattern, + reasoningPatternReason: reasoningPatternSelection.reason, + questionFamily: questionFamilySelection.family, + allowedQuestionFamilies, + rejectedQuestionFamilies, + selectedQuestionTemplate: questionFamilySelection.template, questionComplexity, plainLanguageNormalisations: plainLanguage.normalisations, }; diff --git a/tests/graph/question-formulator.test.js b/tests/graph/question-formulator.test.js index b34ccde..97a8512 100644 --- a/tests/graph/question-formulator.test.js +++ b/tests/graph/question-formulator.test.js @@ -3,6 +3,7 @@ import { assessUnknownAtomicity, formulateQuestion, formulateTieResolutionQuestion, + selectReasoningPattern, selectInvestigationStrategy, } from "@/lib/graph/question-formulator.js"; import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js"; @@ -105,6 +106,8 @@ describe("formulateQuestion", () => { const result = formulateQuestion({ node: unknown, graph }); expect(result.strategy).toBe("decision_threshold"); + expect(result.reasoningPattern).toBe("decision"); + expect(result.questionFamily).toBe("decision_threshold"); expect(result.question).toContain("What outcome"); expect(result.question.toLowerCase()).toContain("justify"); }); @@ -144,6 +147,8 @@ describe("formulateQuestion", () => { }); expect(result.strategy).toBe("definition"); + expect(result.reasoningPattern).toBe("definition"); + expect(result.questionFamily).toBe("definition"); expect(result.question).toMatch(/^What does /); }); @@ -213,9 +218,68 @@ describe("formulateQuestion", () => { }); expect(result.strategy).toBe("contradiction_resolution"); + expect(result.reasoningPattern).toBe("contradiction"); expect(result.question).toContain("resolve the contradiction"); }); + it("reasoning pattern selection marks commercial validation as decision rather than explanation", () => { + const unknown = makeNode({ + id: "n-commercial-pattern", + label: + "Whether the method addresses a genuine, high-priority problem for a specific audience", + description: + "Need to know whether this solves a real problem for a clear audience before continuing development.", + kind: "unknown", + status: "unknown", + confidence: "high", + }); + const graph = makeGraphFor(unknown, { + centralStatement: + "Before investing more, we need to know whether continuing development is commercially justified.", + }); + + const result = selectReasoningPattern({ node: unknown, graph }); + + expect(result.pattern).toBe("decision"); + }); + + it("comparison scenario selects the comparison pattern", () => { + const unknown = makeNode({ + id: "n-comparison-pattern", + label: "How the two observations were measured", + description: + "Need evidence about the measure used for each observation, because that could help explain the difference.", + kind: "unknown", + status: "unknown", + confidence: "medium", + }); + const graph = makeGraphFor(unknown, { + centralStatement: "Traffic increased, but sales stayed flat.", + nodes: [ + makeNode({ + id: "n-traffic-observation", + label: "Traffic increased.", + description: "Traffic increased.", + kind: "observation", + status: "supported", + confidence: "high", + }), + makeNode({ + id: "n-sales-observation", + label: "Sales stayed flat.", + description: "Sales stayed flat.", + kind: "observation", + status: "supported", + confidence: "high", + }), + ], + }); + + const result = selectReasoningPattern({ node: unknown, graph }); + + expect(result.pattern).toBe("comparison"); + }); + it("constraint unknown uses evidence-gathering within the fixed strategy set", () => { const unknown = makeNode({ id: "n-constraint", @@ -445,6 +509,7 @@ describe("formulateQuestion", () => { node: unknown, graph: makeGraphFor(unknown), }); + expect(result.reasoningPattern).toBe("diagnosis"); expect(result.strategy).toBeNull(); expect(result.question).toBe( "What would clarify possible causes of the divergence in this situation?", diff --git a/tests/graph/question-simplicity.test.js b/tests/graph/question-simplicity.test.js index 19c779e..2765346 100644 --- a/tests/graph/question-simplicity.test.js +++ b/tests/graph/question-simplicity.test.js @@ -126,6 +126,11 @@ describe("question simplicity", () => { expect(result.selectedQuestion.question).toBe( "Who experiences this problem?", ); + expect(result.selectedQuestion.reasoningPattern).toBe("decision"); + expect(result.selectedQuestion.questionFamily).toBe("decision_foundation"); + expect(result.selectedQuestion.selectedQuestionTemplate).toBe( + "decision_foundation_direct_child", + ); expect(result.selectedQuestion.question.match(/\?/g) || []).toHaveLength(1); expect(result.questionComplexityAccepted).toBe(true); expect(result.primaryConceptCount).toBe(1); diff --git a/tests/graph/reasoning-pattern-selection.test.js b/tests/graph/reasoning-pattern-selection.test.js new file mode 100644 index 0000000..20eac33 --- /dev/null +++ b/tests/graph/reasoning-pattern-selection.test.js @@ -0,0 +1,201 @@ +import { describe, expect, it } from "vitest"; +import { + formulateQuestion, + formulateTieResolutionQuestion, + selectReasoningPattern, +} from "@/lib/graph/question-formulator.js"; +import { makeGraph, makeNode } from "@/lib/graph/schema.js"; + +function makeGraphFor(node, extra = {}) { + return makeGraph({ + centralStatement: extra.centralStatement || "Decision context", + nodes: [node, ...(extra.nodes || [])], + edges: extra.edges || [], + activeUnknownNodeId: node.id, + resolvedNodeIds: extra.resolvedNodeIds || [], + currentSummary: "Test summary", + reasoningState: extra.reasoningState, + }); +} + +describe("reasoning pattern selection", () => { + it("commercial-method scenario selects the decision pattern and rejects explanation family", () => { + const unknown = makeNode({ + id: "n-commercial-method", + label: + "Whether the method addresses a genuine, high-priority problem for a specific audience", + description: + "Need to know whether this solves a real problem for a clear audience before continuing development.", + kind: "unknown", + status: "unknown", + confidence: "high", + }); + const graph = makeGraphFor(unknown, { + centralStatement: + "Before investing significant time and money, we need to know whether continuing development is commercially justified.", + }); + + const result = formulateQuestion({ node: unknown, graph }); + + expect(result.reasoningPattern).toBe("decision"); + expect(result.questionFamily).not.toBe("explanation"); + expect(result.allowedQuestionFamilies).toContain("decision_foundation"); + expect(result.rejectedQuestionFamilies).toContain("explanation"); + }); + + it("revenue and cash relationship scenario allows the explanation family", () => { + const unknown = makeNode({ + id: "n-revenue-cash-explanation", + label: + "Explanation for why revenue increased by 18%, but cash in the bank fell over the same period", + description: + "Need to understand what change or event could explain why these observations differ.", + kind: "unknown", + status: "unknown", + confidence: "medium", + }); + const graph = makeGraphFor(unknown, { + centralStatement: + "Revenue increased by 18%, but cash in the bank fell over the same period.", + nodes: [ + makeNode({ + id: "n-revenue-observation", + label: "Revenue increased by 18%.", + description: "Revenue increased by 18%.", + kind: "observation", + status: "supported", + confidence: "high", + }), + makeNode({ + id: "n-cash-observation", + label: "Cash in the bank fell over the same period.", + description: "Cash in the bank fell over the same period.", + kind: "observation", + status: "supported", + confidence: "high", + }), + ], + }); + + const result = formulateQuestion({ node: unknown, graph }); + + expect(result.reasoningPattern).toBe("explanation"); + expect(result.allowedQuestionFamilies).toContain("explanation"); + }); + + it("duplicate observations reject explanation family during tie resolution", () => { + const graph = makeGraph({ + centralStatement: "The same figure was repeated twice.", + nodes: [ + makeNode({ + id: "n-obs-1", + label: "Revenue increased by 10%.", + description: "Revenue increased by 10%.", + kind: "observation", + status: "supported", + confidence: "high", + }), + makeNode({ + id: "n-obs-2", + label: "Revenue increased by 10%.", + description: "Revenue increased by 10%.", + kind: "observation", + status: "supported", + confidence: "high", + }), + ], + edges: [], + activeUnknownNodeId: null, + resolvedNodeIds: [], + currentSummary: "Duplicate observation fixture", + }); + + const result = formulateTieResolutionQuestion({ graph }); + + expect(result.questionFamily).not.toBe("explanation"); + expect(result.rejectedQuestionFamilies).toContain("explanation"); + }); + + it("definition scenario selects the definition pattern", () => { + const unknown = makeNode({ + id: "n-definition-pattern", + label: "Definition of justified confidence", + description: "The term needs clearer boundaries.", + kind: "unknown", + status: "unknown", + confidence: "medium", + }); + + const result = selectReasoningPattern({ + node: unknown, + graph: makeGraphFor(unknown), + }); + + expect(result.pattern).toBe("definition"); + }); + + it("comparison scenario selects the comparison pattern", () => { + const unknown = makeNode({ + id: "n-comparison-pattern-2", + label: "How the two observations were measured", + description: + "Need evidence about the measure used for each observation before comparing them.", + kind: "unknown", + status: "unknown", + confidence: "medium", + }); + const graph = makeGraphFor(unknown, { + centralStatement: "Traffic increased, but sales stayed flat.", + nodes: [ + makeNode({ + id: "n-traffic-2", + label: "Traffic increased.", + description: "Traffic increased.", + kind: "observation", + status: "supported", + confidence: "high", + }), + makeNode({ + id: "n-sales-2", + label: "Sales stayed flat.", + description: "Sales stayed flat.", + kind: "observation", + status: "supported", + confidence: "high", + }), + ], + reasoningState: { + comparabilityStatus: "uncertain", + relationshipStatus: "insufficient_information", + }, + }); + + const result = selectReasoningPattern({ node: unknown, graph }); + + expect(result.pattern).toBe("comparison"); + }); + + it("question family stays compatible with the reasoning pattern", () => { + const unknown = makeNode({ + id: "n-decision-family-compatibility", + label: "Who experiences this problem", + description: + "Need to know who experiences this problem before continuing development.", + kind: "unknown", + status: "unknown", + confidence: "medium", + }); + const graph = makeGraphFor(unknown, { + centralStatement: + "We need to know whether continuing development is commercially justified.", + }); + + const result = formulateQuestion({ node: unknown, graph }); + + expect(result.reasoningPattern).toBe("decision"); + expect(result.allowedQuestionFamilies).toContain(result.questionFamily); + expect(result.rejectedQuestionFamilies).not.toContain( + result.questionFamily, + ); + }); +});