From db994d7764963fc7d2930651c17a428c1116b888 Mon Sep 17 00:00:00 2001 From: robbond Date: Mon, 3 Aug 2026 09:13:52 +0100 Subject: [PATCH] fix: make graph-backed questions authoritative --- docs/v0.7-question-simplicity-experiment.md | 42 ++++++++++ lib/graph/apply-proposal.js | 92 +++++++++++++++++++++ lib/graph/orchestrator.js | 91 ++++++++++++++------ tests/graph/orchestrator.test.js | 91 ++++++++++++++++++-- 4 files changed, 285 insertions(+), 31 deletions(-) diff --git a/docs/v0.7-question-simplicity-experiment.md b/docs/v0.7-question-simplicity-experiment.md index a0cb532..635dd2a 100644 --- a/docs/v0.7-question-simplicity-experiment.md +++ b/docs/v0.7-question-simplicity-experiment.md @@ -23,6 +23,8 @@ That violated the intended one-step reasoning discipline. The engine should ask one question about one primary concept at a time. +**The reconstruction model may suggest a question, but only the graph-backed deterministic pipeline may select the user-facing question.** + ## One-question / one-concept rule Every user-facing question should: @@ -85,6 +87,46 @@ This question: - stays graph-backed - avoids pricing or budget before problem existence is established +## Start-case authority rule + +There were previously two question paths during initial analysis: + +- reconstruction model `nextQuestion` +- graph-backed unknown selection and question formulation + +The defect was that `startCase` copied the reconstruction `nextQuestion` directly into the normal UI. + +That path is now closed. + +Initial user-facing questioning now follows this pipeline: + +```text +reconstruction +→ graph build +→ unresolved unknown selection +→ atomicity assessment +→ decomposition if needed +→ investigation strategy +→ question formulation +→ complexity validation +→ selectedQuestion +``` + +The reconstruction question is still retained in diagnostics as provenance, but it is not authoritative. + +## Live result + +Running the commercial-method scenario through the real environment now: + +- succeeds without the enum compatibility failure +- does not show the broad reconstruction question in the UI path +- surfaces a graph-backed first question instead +- keeps the reconstruction question only in diagnostics + +For the tested scenario, the user-facing first question remained: + +> Who experiences this problem? + ## Remaining limitations - question-complexity assessment is still conservative and pattern-based rather than semantic in a richer linguistic sense diff --git a/lib/graph/apply-proposal.js b/lib/graph/apply-proposal.js index da44421..302c56e 100644 --- a/lib/graph/apply-proposal.js +++ b/lib/graph/apply-proposal.js @@ -5,6 +5,7 @@ import { classifyObservationRelationship, COMPARABILITY_REASONING_NODE_ID, formulateQuestion, + formulateTieResolutionQuestion, } from "./question-formulator.js"; import { graphUpdateSchema, @@ -1623,6 +1624,97 @@ function findNodeById(graph, nodeId) { return (graph.nodes || []).find((node) => node.id === nodeId) || null; } +export function determineGraphBackedQuestion({ situationGraph }) { + const graphSnapshot = cloneJsonSafe(situationGraph); + let updatedSituationGraph = cloneJsonSafe(situationGraph); + let deterministicSelection = selectActiveUnknownCandidate( + updatedSituationGraph, + updatedSituationGraph.resolvedNodeIds || [], + ); + + const decompositionResult = runDeterministicDecomposition({ + graphSnapshot, + proposalSnapshot: { + addedNodes: [], + updatedNodes: [], + addedEdges: [], + removedEdgeIds: [], + resolvedUnknownNodeIds: [], + affectedNodeIds: [], + selectedQuestion: null, + }, + updatedSituationGraph, + reasoningResolution: { reasoningStateOverride: {} }, + deterministicSelection, + }); + + if (!decompositionResult.success) { + return decompositionResult; + } + + updatedSituationGraph = decompositionResult.updatedSituationGraph; + updatedSituationGraph.reasoningState = buildReasoningState( + updatedSituationGraph, + ); + deterministicSelection = selectActiveUnknownCandidate( + updatedSituationGraph, + updatedSituationGraph.resolvedNodeIds || [], + ); + updatedSituationGraph.activeUnknownNodeId = + deterministicSelection?.status === "selected" + ? deterministicSelection.nodeId + : null; + updatedSituationGraph.currentSummary = describeGraph(updatedSituationGraph); + + const selectedNode = + deterministicSelection?.status === "selected" + ? findNodeById(updatedSituationGraph, deterministicSelection.nodeId) + : null; + const formulatedQuestion = selectedNode + ? formulateQuestion({ + node: selectedNode, + graph: updatedSituationGraph, + context: { selectionState: deterministicSelection }, + }) + : null; + + return { + success: true, + updatedSituationGraph, + deterministicSelection, + selectedQuestion: + deterministicSelection?.status === "ambiguous" + ? { + id: "q_tie_resolution", + ...formulateTieResolutionQuestion({ graph: updatedSituationGraph }), + nodeId: null, + tiedCandidateIds: deterministicSelection.tiedCandidateIds, + } + : deterministicSelection?.status === "selected" && formulatedQuestion + ? { + nodeId: deterministicSelection.nodeId, + question: + formulatedQuestion.question || deterministicSelection.question, + reason: formulatedQuestion.reason, + strategy: formulatedQuestion.strategy, + investigationStrategy: formulatedQuestion.investigationStrategy, + questionComplexity: formulatedQuestion.questionComplexity, + plainLanguageNormalisations: + formulatedQuestion.plainLanguageNormalisations, + } + : null, + atomicityAssessment: decompositionResult.atomicityAssessment, + decompositionPerformed: + decompositionResult.decompositionAttempted && + decompositionResult.decompositionAccepted, + decompositionAttempted: decompositionResult.decompositionAttempted, + selectedUnknownBefore: decompositionResult.selectedUnknownBefore, + selectedUnknownAfter: deterministicSelection?.nodeId ?? null, + questionComplexityAssessment: + formulatedQuestion?.questionComplexity ?? null, + }; +} + function runDeterministicDecomposition({ graphSnapshot, proposalSnapshot, diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index 1e0b59b..103ec16 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -13,10 +13,14 @@ import { updateCaseRequestSchema, } from "./schema.js"; import { buildInitialGraph, describeGraph } from "./builder.js"; -import { applyValidatedProposal } from "./apply-proposal.js"; +import { + applyValidatedProposal, + determineGraphBackedQuestion, +} from "./apply-proposal.js"; import { buildGraphUpdatePrompt } from "./prompt-builder.js"; import { buildReasoningState, + formulateQuestion, formulateTieResolutionQuestion, } from "./question-formulator.js"; import { parseGraphUpdateProposal } from "./update-proposal.js"; @@ -41,6 +45,13 @@ function buildDiagnostics({ graph, graphReferenceValidation, unknownSelectionExplanation, + reconstructionQuestion, + reconstructionQuestionAccepted, + reconstructionQuestionRejectionReasons, + finalGraphBackedQuestion, + selectedUnknownNodeId, + decompositionApplied, + questionComplexityAssessment, }) { return { promptVersion: analysis?.promptVersion ?? null, @@ -54,6 +65,14 @@ function buildDiagnostics({ compatibilityChanges: analysis?.compatibilityChanges ?? [], compatibilityWarnings: analysis?.compatibilityWarnings ?? [], unknownSelectionExplanation: unknownSelectionExplanation ?? null, + reconstructionQuestion: reconstructionQuestion ?? null, + reconstructionQuestionAccepted: reconstructionQuestionAccepted ?? null, + reconstructionQuestionRejectionReasons: + reconstructionQuestionRejectionReasons ?? [], + finalGraphBackedQuestion: finalGraphBackedQuestion ?? null, + selectedUnknownNodeId: selectedUnknownNodeId ?? null, + decompositionApplied: decompositionApplied ?? false, + questionComplexityAssessment: questionComplexityAssessment ?? null, }; } @@ -263,23 +282,11 @@ export async function startCase(body) { }); const currentSummary = describeGraph(initialGraph); - const deterministicSelection = selectActiveUnknownCandidate( - { - ...initialGraph, - resolvedNodeIds: [], - }, - [], - ); - const activeUnknownNodeId = - deterministicSelection?.status === "selected" - ? deterministicSelection.nodeId - : null; - - const situationGraph = makeGraph({ + const initialSituationGraph = makeGraph({ centralStatement: scenario, nodes: initialGraph.nodes, edges: initialGraph.edges, - activeUnknownNodeId, + activeUnknownNodeId: null, resolvedNodeIds: [], currentSummary, reasoningState: buildReasoningState({ @@ -290,17 +297,20 @@ export async function startCase(body) { }), }); - situationGraphSchema.parse(situationGraph); + situationGraphSchema.parse(initialSituationGraph); - const graphReferenceValidation = validateGraphReferences(situationGraph); - const selectedQuestion = - deterministicSelection?.status === "ambiguous" - ? { - id: "q_tie_resolution", - ...formulateTieResolutionQuestion({ graph: situationGraph }), - tiedCandidateIds: deterministicSelection.tiedCandidateIds, - } - : (analysis.nextQuestion ?? null); + const graphReferenceValidation = validateGraphReferences( + initialSituationGraph, + ); + const initialQuestionResult = determineGraphBackedQuestion({ + situationGraph: initialSituationGraph, + }); + const situationGraph = initialQuestionResult.success + ? initialQuestionResult.updatedSituationGraph + : initialSituationGraph; + const selectedQuestion = initialQuestionResult.success + ? initialQuestionResult.selectedQuestion + : null; const unknownSelectionExplanation = buildUnknownSelectionDiagnostics( situationGraph, [], @@ -315,6 +325,22 @@ export async function startCase(body) { graph: situationGraph, graphReferenceValidation, unknownSelectionExplanation, + reconstructionQuestion: analysis.nextQuestion?.question ?? null, + reconstructionQuestionAccepted: false, + reconstructionQuestionRejectionReasons: + analysis.nextQuestion?.question != null + ? [ + "reconstruction_question_not_authoritative", + "graph_backed_pipeline_required", + ] + : [], + finalGraphBackedQuestion: selectedQuestion?.question ?? null, + selectedUnknownNodeId: + initialQuestionResult.selectedUnknownAfter ?? null, + decompositionApplied: + initialQuestionResult.decompositionPerformed ?? false, + questionComplexityAssessment: + initialQuestionResult.questionComplexityAssessment ?? null, }), validationErrors: graphReferenceValidation.errors, statusCode: 500, @@ -330,6 +356,21 @@ export async function startCase(body) { graph: situationGraph, graphReferenceValidation, unknownSelectionExplanation, + reconstructionQuestion: analysis.nextQuestion?.question ?? null, + reconstructionQuestionAccepted: false, + reconstructionQuestionRejectionReasons: + analysis.nextQuestion?.question != null + ? [ + "reconstruction_question_not_authoritative", + "graph_backed_pipeline_required", + ] + : [], + finalGraphBackedQuestion: selectedQuestion?.question ?? null, + selectedUnknownNodeId: initialQuestionResult.selectedUnknownAfter ?? null, + decompositionApplied: + initialQuestionResult.decompositionPerformed ?? false, + questionComplexityAssessment: + initialQuestionResult.questionComplexityAssessment ?? null, }), }; } diff --git a/tests/graph/orchestrator.test.js b/tests/graph/orchestrator.test.js index 0286d9b..74574d2 100644 --- a/tests/graph/orchestrator.test.js +++ b/tests/graph/orchestrator.test.js @@ -56,6 +56,41 @@ function makeAnalysisResult(overrides = {}) { }; } +function makeCommercialAnalysisResult(overrides = {}) { + return makeAnalysisResult({ + reconstruction: { + summary: + "A new reasoning method may become a commercial product, but problem existence and value remain unresolved.", + actors: [], + systemsOrObjects: [], + expectedStates: [], + observedStates: [], + differences: [], + knownTransitions: [], + unexplainedTransitions: [], + contradictions: [], + importantUnknowns: [ + { + id: "unk-commercial", + label: + "Commercial justification for whether continuing development is commercially justified", + description: + "Need to know whether this solves a genuine problem, whether people would value it enough to pay for it, and whether it is commercially justified before continuing development.", + confidence: "high", + }, + ], + plausibleInterpretations: [], + }, + nextQuestion: { + id: "q-commercial", + question: + "What specific validation metrics, pilot feedback, or competitive benchmarking results have you collected to measure whether the method solves a recognized problem and how target users evaluate its practical utility compared to existing tools?", + reason: "Model-proposed broad validation question", + }, + ...overrides, + }); +} + function makeUpdateGraph() { const unknown = makeNode({ id: "n-unknown", @@ -401,9 +436,15 @@ describe("lib/graph/orchestrator startCase", () => { }); }); - it("returns null selectedQuestion when analysis has no nextQuestion", async () => { + it("returns null selectedQuestion when neither analysis nor graph path yields a question", async () => { mockAnalyseScenario.mockResolvedValue( - makeAnalysisResult({ nextQuestion: undefined }), + makeAnalysisResult({ + nextQuestion: undefined, + reconstruction: { + ...makeAnalysisResult().reconstruction, + importantUnknowns: [], + }, + }), ); const { startCase } = await import("@/lib/graph/orchestrator.js"); @@ -413,6 +454,40 @@ describe("lib/graph/orchestrator startCase", () => { expect(result.selectedQuestion).toBeNull(); }); + it("uses the graph-backed question path instead of the reconstruction nextQuestion on startCase", async () => { + mockAnalyseScenario.mockResolvedValue(makeCommercialAnalysisResult()); + const { startCase } = await import("@/lib/graph/orchestrator.js"); + + const result = await startCase({ + scenario: + "I have developed a new reasoning method that aims to help people determine whether they have enough justified confidence to make a decision.", + }); + + expect(result.success).toBe(true); + expect(result.selectedQuestion?.question).toBe( + "Who experiences this problem?", + ); + expect(result.selectedQuestion?.nodeId).toBe( + result.situationGraph.activeUnknownNodeId, + ); + expect(result.selectedQuestion?.question).not.toContain( + "validation metrics, pilot feedback, or competitive benchmarking", + ); + expect(result.diagnostics.reconstructionQuestion).toContain( + "validation metrics, pilot feedback, or competitive benchmarking", + ); + expect(result.diagnostics.reconstructionQuestionAccepted).toBe(false); + expect(result.diagnostics.reconstructionQuestionRejectionReasons).toContain( + "graph_backed_pipeline_required", + ); + expect(result.diagnostics.finalGraphBackedQuestion).toBe( + "Who experiences this problem?", + ); + expect(result.diagnostics.selectedUnknownNodeId).toBe( + result.situationGraph.activeUnknownNodeId, + ); + }); + it("includes compatibility diagnostics when provided by analysis", async () => { mockAnalyseScenario.mockResolvedValue( makeAnalysisResult({ @@ -1093,16 +1168,20 @@ describe("lib/graph/orchestrator startCase", () => { ); }); - it("startCase behaviour remains unchanged", async () => { + it("startCase no longer copies analysis nextQuestion directly when a graph-backed question exists", async () => { mockAnalyseScenario.mockResolvedValue(makeAnalysisResult()); const { startCase } = await import("@/lib/graph/orchestrator.js"); const result = await startCase({ scenario: "Scenario text" }); expect(result.success).toBe(true); - expect(result.selectedQuestion).toEqual({ - id: "q-1", - question: "What denominator is being used for the complaint rate?", + expect(result.selectedQuestion).toMatchObject({ + nodeId: result.situationGraph.activeUnknownNodeId, + question: + "What would clarify need the denominator for complaint rate in this situation?", }); + expect(result.diagnostics.reconstructionQuestion).toBe( + "What denominator is being used for the complaint rate?", + ); }); });