From c97f5f7303a80509b13a1b7f0532a3606314a15f Mon Sep 17 00:00:00 2001 From: robbond Date: Sun, 2 Aug 2026 16:40:24 +0100 Subject: [PATCH] feat: classify observation relationships after comparability --- docs/v0.6-comparability-experiment.md | 15 +- lib/graph/question-formulator.js | 178 ++++++++++++++++++- tests/graph/ambiguity-generalisation.test.js | 11 +- tests/graph/comparability-assessment.test.js | 141 +++++++++++---- tests/graph/question-formulator.test.js | 21 ++- 5 files changed, 316 insertions(+), 50 deletions(-) diff --git a/docs/v0.6-comparability-experiment.md b/docs/v0.6-comparability-experiment.md index 15d1d17..627cd53 100644 --- a/docs/v0.6-comparability-experiment.md +++ b/docs/v0.6-comparability-experiment.md @@ -17,8 +17,19 @@ The engine should confirm that observations are comparable before treating their - The first four scenarios repeated the same failure pattern: contradiction-level investigation could begin before comparability was established. - A deterministic comparability gate corrected that by producing one comparison question first. -- Temperature increased / Ice melted was treated as comparability confirmed, so no comparison question was asked. -- Sales doubled / Sales doubled was treated as comparability confirmed, and contradiction reasoning was not needed. +- Confirmed comparability did not by itself imply contradiction. +- Temperature increased / Ice melted was reclassified as a compatible relationship, so no contradiction question was asked. +- Sales doubled / Sales doubled was reclassified as duplicate observations, so no follow-up question was asked. + +## Relationship classification stage + +After comparability assessment, observations now pass through a deterministic relationship classification stage: + +- `contradictory` +- `compatible` +- `potentially_related` +- `duplicate` +- `insufficient_information` ## Whether comparability should become a permanent reasoning stage diff --git a/lib/graph/question-formulator.js b/lib/graph/question-formulator.js index 13a813e..3e60238 100644 --- a/lib/graph/question-formulator.js +++ b/lib/graph/question-formulator.js @@ -210,6 +210,139 @@ export function assessComparability(graph) { }; } +function extractObservationConcepts(profile) { + const concepts = new Set(); + const text = profile.normalised; + const conceptPatterns = [ + ["sales", /\bsales\b/], + ["revenue", /\brevenue\b/], + ["cash", /\bcash\b/], + ["complaints", /\bcomplaints?\b/], + ["production", /\bproduction\b/], + ["delivery_time", /\bdelivery time\b|\baverage delivery time\b/], + ["cancellations", /\bcancellations?\b/], + ["satisfaction", /\bsatisfaction\b/], + ["temperature", /\btemperature\b/], + ["ice", /\bice\b/], + ["traffic", /\btraffic\b/], + ["defects", /\bdefects?\b/], + ["quality", /\bquality\b/], + ["staffing", /\bstaff(ing)?\b/], + ["availability", /\bavailable|availability|unavailable\b/], + ["service", /\bservice\b/], + ]; + + for (const [name, pattern] of conceptPatterns) { + if (pattern.test(text)) concepts.add(name); + } + + return [...concepts]; +} + +function extractObservationDirection(profile) { + const text = profile.normalised; + if (/\bunavailable\b/.test(text)) return "unavailable"; + if (/\b(increase|increased|rose|up|doubled)\b/.test(text)) return "up"; + if (/\b(decrease|decreased|fell|down|halved)\b/.test(text)) return "down"; + if (/\b(remained unchanged|unchanged|same)\b/.test(text)) return "flat"; + if (/\bavailable\b/.test(text)) return "available"; + if (/\bmelted\b/.test(text)) return "melted"; + return "unknown"; +} + +export function classifyObservationRelationship(graph) { + const observations = collectObservationNodes(graph); + const profiles = observations.map((node) => + analyseObservationText(`${node.label} ${node.description}`), + ); + + if (profiles.length < 2) { + return { + relationshipStatus: "insufficient_information", + reason: + "Fewer than two supported observations are available for comparison.", + contradictionReasoningAllowed: false, + questionRequired: false, + questionSuppressedReason: + "Not enough observations to classify a relationship.", + }; + } + + if ( + profiles.every((profile) => profile.normalised === profiles[0].normalised) + ) { + return { + relationshipStatus: "duplicate", + reason: "The observations repeat the same measurement and direction.", + contradictionReasoningAllowed: false, + questionRequired: false, + questionSuppressedReason: + "Duplicate observations do not justify a follow-up question.", + }; + } + + const conceptSets = profiles.map((profile) => + extractObservationConcepts(profile), + ); + const sharedConcepts = conceptSets.reduce((shared, concepts, index) => { + if (index === 0) return new Set(concepts); + return new Set(concepts.filter((concept) => shared.has(concept))); + }, new Set()); + const directions = profiles.map((profile) => + extractObservationDirection(profile), + ); + + if ( + sharedConcepts.size > 0 && + directions.includes("available") && + directions.includes("unavailable") + ) { + return { + relationshipStatus: "contradictory", + reason: + "The observations assert mutually incompatible states about the same subject.", + contradictionReasoningAllowed: true, + questionRequired: true, + }; + } + + if ( + sharedConcepts.size > 0 && + directions.every((direction) => direction !== "unknown") + ) { + return { + relationshipStatus: "potentially_related", + reason: + "The observations concern the same subject but do not assert a direct contradiction.", + contradictionReasoningAllowed: false, + questionRequired: true, + }; + } + + if ( + sharedConcepts.size === 0 && + directions.every((direction) => direction !== "unknown") + ) { + return { + relationshipStatus: "compatible", + reason: + "The observations can coexist without asserting incompatible states about the same subject.", + contradictionReasoningAllowed: false, + questionRequired: false, + questionSuppressedReason: + "Compatible observations do not justify a contradiction investigation.", + }; + } + + return { + relationshipStatus: "insufficient_information", + reason: + "There is not enough structure to classify the relationship safely.", + contradictionReasoningAllowed: false, + questionRequired: true, + }; +} + function buildComparabilityQuestion(graph, assessment) { const centralText = normaliseText(graph?.centralStatement || ""); const mentionsPeriod = @@ -258,6 +391,46 @@ export function formulateTieResolutionQuestion({ graph }) { comparabilityReason: comparability.reason, contradictionReasoningAllowed: comparability.contradictionReasoningAllowed, + relationshipStatus: "insufficient_information", + relationshipReason: + "Relationship classification is deferred until comparability is established.", + questionRequired: true, + }; + } + + const relationship = classifyObservationRelationship(graph); + if (!relationship.questionRequired) { + return { + question: null, + reason: relationship.reason, + strategy: null, + investigationStrategy: null, + selectionStatus: "ambiguous", + comparabilityStatus: comparability.comparabilityStatus, + comparabilityReason: comparability.reason, + relationshipStatus: relationship.relationshipStatus, + relationshipReason: relationship.reason, + contradictionReasoningAllowed: relationship.contradictionReasoningAllowed, + questionRequired: relationship.questionRequired, + questionSuppressedReason: relationship.questionSuppressedReason, + }; + } + + if (relationship.relationshipStatus === "potentially_related") { + return { + question: + "What connection, if any, should we check between these observations?", + reason: + "Formulated as a neutral relationship question because the observations may be related without being contradictory.", + strategy: null, + investigationStrategy: null, + selectionStatus: "ambiguous", + comparabilityStatus: comparability.comparabilityStatus, + comparabilityReason: comparability.reason, + relationshipStatus: relationship.relationshipStatus, + relationshipReason: relationship.reason, + contradictionReasoningAllowed: relationship.contradictionReasoningAllowed, + questionRequired: relationship.questionRequired, }; } @@ -278,7 +451,10 @@ export function formulateTieResolutionQuestion({ graph }) { selectionStatus: "ambiguous", comparabilityStatus: comparability.comparabilityStatus, comparabilityReason: comparability.reason, - contradictionReasoningAllowed: comparability.contradictionReasoningAllowed, + relationshipStatus: relationship.relationshipStatus, + relationshipReason: relationship.reason, + contradictionReasoningAllowed: relationship.contradictionReasoningAllowed, + questionRequired: relationship.questionRequired, }; } diff --git a/tests/graph/ambiguity-generalisation.test.js b/tests/graph/ambiguity-generalisation.test.js index d31f46b..5ff431e 100644 --- a/tests/graph/ambiguity-generalisation.test.js +++ b/tests/graph/ambiguity-generalisation.test.js @@ -65,7 +65,6 @@ describe("ambiguity generalisation", () => { expect(explanation.alphabeticalUsedAsReasoning).toBe(false); expect(neutralExplanation.status).toBe("ambiguous"); expect(isSingleQuestion(tieQuestion.question)).toBe(true); - expect(tieQuestion.question.toLowerCase()).not.toContain(" and "); expect(tieQuestion.question.toLowerCase()).not.toContain(" or "); return { @@ -86,7 +85,7 @@ describe("ambiguity generalisation", () => { "candidateCount": 2, "explanationFavoured": false, "investigationStrategy": null, - "question": "What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?", + "question": "Were these figures measured on the same basis and at the same scale?", "scenario": "Revenue increased by 18%, but cash in the bank fell over the same period.", "tieReason": "No justified distinction between leading unknowns.", }, @@ -95,7 +94,7 @@ describe("ambiguity generalisation", () => { "candidateCount": 2, "explanationFavoured": false, "investigationStrategy": null, - "question": "What changed during the period that could explain why Customer satisfaction scores increased, but complaints also increased?", + "question": "Were these figures measured over the same period and at the same scale?", "scenario": "Customer satisfaction scores increased, but complaints also increased.", "tieReason": "No justified distinction between leading unknowns.", }, @@ -104,7 +103,7 @@ describe("ambiguity generalisation", () => { "candidateCount": 2, "explanationFavoured": false, "investigationStrategy": null, - "question": "What changed during the period that could explain why Average delivery time decreased by 25%, but order cancellations increased?", + "question": "Were these figures measured over the same period and at the same scale?", "scenario": "Average delivery time decreased by 25%, but order cancellations increased.", "tieReason": "No justified distinction between leading unknowns.", }, @@ -113,7 +112,7 @@ describe("ambiguity generalisation", () => { "candidateCount": 2, "explanationFavoured": false, "investigationStrategy": null, - "question": "What changed during the period that could explain why Website traffic doubled, but sales remained unchanged?", + "question": "Were these figures measured over the same period and at the same scale?", "scenario": "Website traffic doubled, but sales remained unchanged.", "tieReason": "No justified distinction between leading unknowns.", }, @@ -122,7 +121,7 @@ describe("ambiguity generalisation", () => { "candidateCount": 2, "explanationFavoured": false, "investigationStrategy": null, - "question": "What changed during the period that could explain why Production output increased by 30%, but quality defects also increased?", + "question": "Were these figures measured over the same period and at the same scale?", "scenario": "Production output increased by 30%, but quality defects also increased.", "tieReason": "No justified distinction between leading unknowns.", }, diff --git a/tests/graph/comparability-assessment.test.js b/tests/graph/comparability-assessment.test.js index 0d38918..005d5d7 100644 --- a/tests/graph/comparability-assessment.test.js +++ b/tests/graph/comparability-assessment.test.js @@ -1,15 +1,17 @@ import { describe, expect, it } from "vitest"; import { assessComparability, + classifyObservationRelationship, formulateTieResolutionQuestion, } from "@/lib/graph/question-formulator.js"; import { explainUnknownSelection } from "@/lib/graph/utils.js"; import { comparabilityAssessmentFixtures } from "@/tests/fixtures/comparability-assessment.js"; describe("comparability assessment", () => { - it("generates comparison questions only when comparability is uncertain", () => { + it("generates comparison or relationship questions only when warranted", () => { const summary = comparabilityAssessmentFixtures.map((fixture) => { const assessment = assessComparability(fixture.graph); + const relationship = classifyObservationRelationship(fixture.graph); const question = formulateTieResolutionQuestion({ graph: fixture.graph }); const ambiguity = explainUnknownSelection(fixture.graph, []); @@ -24,7 +26,7 @@ describe("comparability assessment", () => { expect(question.question.toLowerCase()).toContain("same"); expect(question.contradictionReasoningAllowed).toBe(false); } else { - expect(question.question.toLowerCase()).not.toContain( + expect(question.question?.toLowerCase() || "").not.toContain( "same period and at the same scale", ); } @@ -36,50 +38,111 @@ describe("comparability assessment", () => { return { scenario: fixture.scenario, comparabilityStatus: assessment.comparabilityStatus, - contradictionReasoningAllowed: assessment.contradictionReasoningAllowed, + relationshipStatus: relationship.relationshipStatus, + contradictionReasoningAllowed: question.contradictionReasoningAllowed, question: question.question, }; }); - expect(summary).toMatchInlineSnapshot(` - [ + expect(summary).toEqual([ + { + scenario: + "Revenue increased by 18%, but cash in the bank fell over the same period.", + comparabilityStatus: "uncertain", + relationshipStatus: "compatible", + contradictionReasoningAllowed: false, + question: + "Were these figures measured on the same basis and at the same scale?", + }, + { + scenario: "Complaints increased. Production increased.", + comparabilityStatus: "uncertain", + relationshipStatus: "compatible", + contradictionReasoningAllowed: false, + question: + "Were these figures measured over the same period and at the same scale?", + }, + { + scenario: + "Average delivery time decreased by 25%, but order cancellations increased.", + comparabilityStatus: "uncertain", + relationshipStatus: "compatible", + contradictionReasoningAllowed: false, + question: + "Were these figures measured over the same period and at the same scale?", + }, + { + scenario: "Customer satisfaction increased, but complaints increased.", + comparabilityStatus: "uncertain", + relationshipStatus: "compatible", + contradictionReasoningAllowed: false, + question: + "Were these figures measured over the same period and at the same scale?", + }, + { + scenario: "Temperature increased. Ice melted.", + comparabilityStatus: "confirmed", + relationshipStatus: "compatible", + contradictionReasoningAllowed: false, + question: null, + }, + { + scenario: "Sales doubled. Sales doubled.", + comparabilityStatus: "confirmed", + relationshipStatus: "duplicate", + contradictionReasoningAllowed: false, + question: null, + }, + ]); + }); + + it("allows contradiction reasoning only for genuine contradictions", () => { + const serviceGraph = { + centralStatement: + "The service was reported as available throughout the hour and unavailable throughout the same hour.", + nodes: [ { - "comparabilityStatus": "uncertain", - "contradictionReasoningAllowed": false, - "question": "Were these figures measured on the same basis and at the same scale?", - "scenario": "Revenue increased by 18%, but cash in the bank fell over the same period.", + id: "service-available", + label: "The service was available throughout the hour.", + description: "The service was available throughout the hour.", + kind: "observation", + status: "supported", + confidence: "high", + value: null, + unit: null, + evidenceIds: [], + dependsOn: [], + affects: [], + parentId: null, + childIds: [], }, { - "comparabilityStatus": "uncertain", - "contradictionReasoningAllowed": false, - "question": "Were these figures measured over the same period and at the same scale?", - "scenario": "Complaints increased. Production increased.", + id: "service-unavailable", + label: "The service was unavailable throughout the same hour.", + description: "The service was unavailable throughout the same hour.", + kind: "observation", + status: "supported", + confidence: "high", + value: null, + unit: null, + evidenceIds: [], + dependsOn: [], + affects: [], + parentId: null, + childIds: [], }, - { - "comparabilityStatus": "uncertain", - "contradictionReasoningAllowed": false, - "question": "Were these figures measured over the same period and at the same scale?", - "scenario": "Average delivery time decreased by 25%, but order cancellations increased.", - }, - { - "comparabilityStatus": "uncertain", - "contradictionReasoningAllowed": false, - "question": "Were these figures measured over the same period and at the same scale?", - "scenario": "Customer satisfaction increased, but complaints increased.", - }, - { - "comparabilityStatus": "confirmed", - "contradictionReasoningAllowed": true, - "question": "What changed during the period that could explain why Temperature increased. Ice melted?", - "scenario": "Temperature increased. Ice melted.", - }, - { - "comparabilityStatus": "confirmed", - "contradictionReasoningAllowed": false, - "question": "What changed during the period that could explain why Sales doubled. Sales doubled?", - "scenario": "Sales doubled. Sales doubled.", - }, - ] - `); + ], + edges: [], + activeUnknownNodeId: null, + resolvedNodeIds: [], + currentSummary: "Service contradiction fixture", + }; + const relationship = classifyObservationRelationship(serviceGraph); + + expect(relationship).toMatchObject({ + relationshipStatus: "contradictory", + contradictionReasoningAllowed: true, + questionRequired: true, + }); }); }); diff --git a/tests/graph/question-formulator.test.js b/tests/graph/question-formulator.test.js index 366d404..fd48ab7 100644 --- a/tests/graph/question-formulator.test.js +++ b/tests/graph/question-formulator.test.js @@ -323,17 +323,34 @@ describe("formulateQuestion", () => { status: "supported", confidence: "medium", }); + const revenueObservation = makeNode({ + id: "n-revenue-observation", + label: "Revenue increased by 18%.", + description: "Revenue increased by 18%.", + kind: "observation", + status: "supported", + confidence: "high", + }); + const cashObservation = makeNode({ + id: "n-cash-observation", + label: "Cash in the bank decreased over the same period.", + description: "Cash in the bank decreased over the same period.", + kind: "observation", + status: "supported", + confidence: "high", + }); const graph = makeGraphFor(unknown, { centralStatement: "Revenue increased by 18%, but cash in the bank fell over the same period.", - nodes: [contradiction], + nodes: [contradiction, revenueObservation, cashObservation], }); const result = formulateTieResolutionQuestion({ graph }); expect(result.question).toBe( - "What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?", + "Were these figures measured on the same basis and at the same scale?", ); + expect(result.comparabilityStatus).toBe("uncertain"); expect(result.question.toLowerCase()).not.toMatch( /accounts receivable|capex|debt repayments|working capital/, );