From 4e4d0fa7329f71226d6f3ecaf24f8f1f502c2298 Mon Sep 17 00:00:00 2001 From: robbond Date: Sun, 9 Aug 2026 19:55:35 +0100 Subject: [PATCH] reasoning: stop answer fidelity guard blocking valid unclassified answers --- lib/graph/apply-proposal.js | 89 ++++++++++++++++++++- tests/graph/apply-proposal.test.js | 121 ++++++++++++++++++++++++++++- 2 files changed, 207 insertions(+), 3 deletions(-) diff --git a/lib/graph/apply-proposal.js b/lib/graph/apply-proposal.js index 1c99be4..0ad7b78 100644 --- a/lib/graph/apply-proposal.js +++ b/lib/graph/apply-proposal.js @@ -2689,6 +2689,70 @@ function normaliseSemanticText(value) { .trim(); } +function semanticContentTokens(value) { + const stopWords = new Set([ + "the", + "and", + "for", + "that", + "this", + "with", + "from", + "into", + "than", + "then", + "they", + "them", + "their", + "there", + "about", + "would", + "could", + "should", + "because", + "being", + "been", + "have", + "has", + "had", + "were", + "what", + "when", + "where", + "which", + "while", + "mainly", + "roughly", + "specifically", + "directly", + "user", + ]); + + return normaliseSemanticText(value) + .replace(/[^a-z0-9]+/g, " ") + .split(" ") + .filter((token) => token.length > 2 && !stopWords.has(token)); +} + +function semanticOverlapRatio(sourceText, candidateText) { + const source = new Set(semanticContentTokens(sourceText)); + const candidate = new Set(semanticContentTokens(candidateText)); + + if (candidate.size === 0) return 1; + + const overlap = [...candidate].filter((token) => source.has(token)).length; + return overlap / candidate.size; +} + +function rawAnswerSupportsUnclassifiedMeaning(answer, userSupportedMeaning) { + const overlapRatio = semanticOverlapRatio(answer, userSupportedMeaning); + const overlappingTokens = semanticContentTokens(userSupportedMeaning).filter( + (token) => semanticContentTokens(answer).includes(token), + ).length; + + return overlapRatio >= 0.4 || overlappingTokens >= 3; +} + function hasConditionalQualification(text) { const value = normaliseSemanticText(text); return ( @@ -2893,6 +2957,23 @@ function validateAnswerMeaningCompatibilityWithRawAnswer({ answer, proposal }) { } } + if (rawAnswerProfile.category === "other") { + if (supportedMeaningProfile.category !== "other") { + errors.push( + "answerMeaning.userSupportedMeaning introduces a stronger reasoning category than the raw answer establishes.", + ); + } else if ( + !rawAnswerSupportsUnclassifiedMeaning( + answer, + proposal.answerMeaning.userSupportedMeaning, + ) + ) { + errors.push( + "answerMeaning.userSupportedMeaning introduces unsupported meaning beyond what the raw answer itself states.", + ); + } + } + return errors; } @@ -2964,9 +3045,13 @@ function validateAnswerMeaningAlignment(proposal) { } if (supportCategory === "other") { - if (resolved || containsConstraintBoundaryLanguage(proposalText)) { + if ( + resolved && + containsConstraintBoundaryLanguage(proposalText) && + !containsConstraintBoundaryLanguage(meaningText) + ) { errors.push( - "Proposal cannot resolve or strengthen answerMeaning that does not clearly establish one of the protected reasoning categories.", + "Proposal cannot resolve beyond an unclassified answer by introducing an unsupported constraint or preference/trade-off distinction.", ); } } diff --git a/tests/graph/apply-proposal.test.js b/tests/graph/apply-proposal.test.js index 08510ed..cb245a7 100644 --- a/tests/graph/apply-proposal.test.js +++ b/tests/graph/apply-proposal.test.js @@ -1475,6 +1475,125 @@ describe("applyValidatedProposal", () => { ); }); + it("57A regression: a legitimate decision-advancing answer classified as other may resolve its intended unknown", () => { + const graph = makeCommercialUpdateFixture(); + const parentId = graph.activeUnknownNodeId; + + const result = applyValidatedProposal({ + situationGraph: graph, + answer: + "We're looking at this mainly for cost reduction — roughly £2M annual savings on office overhead.", + previousQuestion: + "What problem would this need to solve to justify continuing development?", + proposal: { + addedNodes: [ + makeNode({ + id: "n-other-people-problem", + label: "Whether other people experience this problem", + description: + "Need to know whether other people experience this problem, because that must be established before deciding whether the problem is broadly important.", + kind: "unknown", + status: "unknown", + confidence: "medium", + parentId, + }), + ], + updatedNodes: [ + { + nodeId: parentId, + previousStatus: "unknown", + newStatus: "resolved", + previousValue: null, + newValue: + "A concrete problem exists: reducing office overhead by roughly £2M annually is the main outcome being sought.", + reason: + "The answer directly states the practical decision-driving outcome the work is intended to achieve.", + }, + ], + addedEdges: [ + makeEdge({ + id: "e-commercial-parent-other-people-problem", + fromNodeId: parentId, + toNodeId: "n-other-people-problem", + relationship: "depends_on", + confidence: "medium", + description: + "After establishing the concrete problem in the current context, the next unknown is whether it also exists for other people.", + }), + ], + removedEdgeIds: [], + resolvedUnknownNodeIds: [parentId], + affectedNodeIds: [], + selectedQuestion: { + nodeId: "n-other-people-problem", + question: + "What makes you think other people experience this problem too?", + reason: + "The answer establishes the problem in this case; the next consequential unknown is whether it generalises beyond this case.", + }, + answerMeaning: { + userSupportedMeaning: + "The main reason for considering this is cost reduction, specifically about £2M in annual office-overhead savings.", + possibleInference: + "If those savings are real and recurring, that could make the problem commercially important.", + supportCategory: "other", + resolutionGuidance: null, + }, + }, + }); + + expect(result.success).toBe(true); + expect(result.updatedSituationGraph.resolvedNodeIds).toContain(parentId); + expect(result.stage).toBeUndefined(); + }); + + it("unsafe unclassified answer: other is not automatically trusted when the proposal adds stronger unsupported meaning", () => { + const graph = makeCommercialUpdateFixture(); + const parentId = graph.activeUnknownNodeId; + + const result = applyValidatedProposal({ + situationGraph: graph, + answer: + "We're looking at this mainly for cost reduction — roughly £2M annual savings on office overhead.", + previousQuestion: + "What problem would this need to solve to justify continuing development?", + proposal: { + addedNodes: [], + updatedNodes: [ + { + nodeId: parentId, + previousStatus: "unknown", + newStatus: "resolved", + previousValue: null, + newValue: + "There is proven broad market demand for a product that delivers these savings.", + reason: + "The answer was treated as establishing commercial demand rather than only the user's own cost-reduction goal.", + }, + ], + addedEdges: [], + removedEdgeIds: [], + resolvedUnknownNodeIds: [parentId], + affectedNodeIds: [], + selectedQuestion: null, + answerMeaning: { + userSupportedMeaning: + "There is proven broad market demand for a product that delivers these savings.", + possibleInference: + "The savings target could imply broader applicability if others share similar overhead pressures.", + supportCategory: "other", + resolutionGuidance: null, + }, + }, + }); + + expect(result.success).toBe(false); + expect(result.stage).toBe("proposal_compatibility"); + expect(result.errors.join(" ")).toContain( + "unsupported meaning beyond what the raw answer itself states", + ); + }); + it("Regression C: rejects unresolved uncertainty being treated as resolved", () => { const { graph, riskUnknownId } = makeRiskClarificationFixture(); @@ -1690,7 +1809,7 @@ describe("applyValidatedProposal", () => { expect(result.success).toBe(false); expect(result.stage).toBe("proposal_compatibility"); expect(result.errors.join(" ")).toContain( - "does not clearly establish one of the protected reasoning categories", + "unsupported constraint or preference/trade-off distinction", ); });