diff --git a/docs/v0.6-comparability-experiment.md b/docs/v0.6-comparability-experiment.md new file mode 100644 index 0000000..15d1d17 --- /dev/null +++ b/docs/v0.6-comparability-experiment.md @@ -0,0 +1,27 @@ +# v0.6 Comparability Experiment + +## Hypothesis + +The engine should confirm that observations are comparable before treating their difference as a contradiction that needs explanatory follow-up. + +## Fixtures + +1. Revenue increased by 18%, but cash in the bank fell over the same period. +2. Complaints increased. Production increased. +3. Average delivery time decreased by 25%, but order cancellations increased. +4. Customer satisfaction increased, but complaints increased. +5. Temperature increased. Ice melted. +6. Sales doubled. Sales doubled. + +## Results + +- The first four scenarios repeated the same failure pattern: contradiction-level investigation could begin before comparability was established. +- A deterministic comparability gate corrected that by producing one comparison question first. +- Temperature increased / Ice melted was treated as comparability confirmed, so no comparison question was asked. +- Sales doubled / Sales doubled was treated as comparability confirmed, and contradiction reasoning was not needed. + +## Whether comparability should become a permanent reasoning stage + +Yes, in minimal deterministic form. + +The repeated pattern appeared in four scenarios, so a small pre-contradiction comparability assessment is justified. diff --git a/lib/graph/question-formulator.js b/lib/graph/question-formulator.js index e9ebad8..13a813e 100644 --- a/lib/graph/question-formulator.js +++ b/lib/graph/question-formulator.js @@ -123,6 +123,105 @@ function buildEvidenceFallbackQuestion(meaning) { return `What evidence would confirm or rule out ${stripTrailingPunctuation(meaning)}?`; } +function collectObservationNodes(graph) { + return (graph?.nodes || []).filter( + (node) => node.kind === "observation" && node.status === "supported", + ); +} + +function analyseObservationText(text) { + const normalised = normaliseText(text); + return { + text, + normalised, + isMeasurementLike: + /\b(increase|increased|decrease|decreased|fell|rose|doubled|halved|remained|average|score|scores|rate|time|traffic|sales|output|defects|complaints|production|revenue|cash|temperature|quality)\b/.test( + normalised, + ) || /%|percent/.test(String(text || "")), + timeframeMentioned: + /\b(period|timeframe|quarter|month|week|year|day|annual|daily|weekly|monthly|same period)\b/.test( + normalised, + ), + scaleMentioned: /\b(average|rate|score|scores|per|percent|%)\b/.test( + normalised, + ), + unitMentioned: + /\b(celsius|fahrenheit|minutes|minute|hours|hour|days|day|units|sales|traffic|cash|revenue|complaints|defects)\b/.test( + normalised, + ), + }; +} + +export function assessComparability(graph) { + const observations = collectObservationNodes(graph); + const centralText = normaliseText(graph?.centralStatement || ""); + const profiles = observations.map((node) => + analyseObservationText(`${node.label} ${node.description}`), + ); + + if (profiles.length < 2) { + return { + comparabilityStatus: "confirmed", + reason: "Fewer than two supported observations need comparison.", + contradictionReasoningAllowed: true, + }; + } + + if ( + profiles.every((profile) => profile.normalised === profiles[0].normalised) + ) { + return { + comparabilityStatus: "confirmed", + reason: "The observations restate the same measurement.", + contradictionReasoningAllowed: false, + }; + } + + if (profiles.some((profile) => !profile.isMeasurementLike)) { + return { + comparabilityStatus: "confirmed", + reason: "The observations are not competing like-for-like measurements.", + contradictionReasoningAllowed: true, + }; + } + + const hasExplicitTimeframe = + /\b(period|timeframe|quarter|month|week|year|day|same period)\b/.test( + centralText, + ) || profiles.every((profile) => profile.timeframeMentioned); + + const hasSharedScale = profiles.every((profile) => profile.scaleMentioned); + const hasSharedUnits = profiles.every((profile) => profile.unitMentioned); + + if (!hasExplicitTimeframe || !hasSharedScale || !hasSharedUnits) { + return { + comparabilityStatus: "uncertain", + reason: + "Comparability between the observations is not yet established across period, scale, or measurement basis.", + contradictionReasoningAllowed: false, + }; + } + + return { + comparabilityStatus: "uncertain", + reason: + "The observations appear comparable in form, but the basis for comparing them is still not established.", + contradictionReasoningAllowed: false, + }; +} + +function buildComparabilityQuestion(graph, assessment) { + const centralText = normaliseText(graph?.centralStatement || ""); + const mentionsPeriod = + /\b(period|timeframe|quarter|month|week|year|day)\b/.test(centralText); + + if (mentionsPeriod) { + return "Were these figures measured on the same basis and at the same scale?"; + } + + return "Were these figures measured over the same period and at the same scale?"; +} + function detectContradictionContext(graph) { const central = stripTrailingPunctuation( graph?.centralStatement || "this situation", @@ -146,6 +245,22 @@ function detectContradictionContext(graph) { } export function formulateTieResolutionQuestion({ graph }) { + const comparability = assessComparability(graph); + if (comparability.comparabilityStatus === "uncertain") { + return { + question: buildComparabilityQuestion(graph, comparability), + reason: + "Formulated to confirm whether the observations are comparable before exploring competing explanations.", + strategy: null, + investigationStrategy: null, + selectionStatus: "ambiguous", + comparabilityStatus: comparability.comparabilityStatus, + comparabilityReason: comparability.reason, + contradictionReasoningAllowed: + comparability.contradictionReasoningAllowed, + }; + } + const { centralStatement, contradictionLabel } = detectContradictionContext(graph); const focus = @@ -161,6 +276,9 @@ export function formulateTieResolutionQuestion({ graph }) { strategy: null, investigationStrategy: null, selectionStatus: "ambiguous", + comparabilityStatus: comparability.comparabilityStatus, + comparabilityReason: comparability.reason, + contradictionReasoningAllowed: comparability.contradictionReasoningAllowed, }; } diff --git a/tests/fixtures/comparability-assessment.js b/tests/fixtures/comparability-assessment.js new file mode 100644 index 0000000..db64b6b --- /dev/null +++ b/tests/fixtures/comparability-assessment.js @@ -0,0 +1,164 @@ +import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js"; + +function buildComparabilityFixture({ + key, + scenario, + observationLabels, + contradictionLabel, + expectedComparabilityStatus, + expectsComparisonQuestion, +}) { + const summary = makeNode({ + id: `${key}-summary`, + label: scenario, + description: "Summary of the situation from the scenario text", + kind: "state", + status: "provisional", + confidence: "medium", + }); + + const observations = observationLabels.map((label, index) => + makeNode({ + id: `${key}-obs-${index + 1}`, + label, + description: label, + kind: "observation", + status: "supported", + confidence: "high", + }), + ); + + const contradiction = contradictionLabel + ? [ + makeNode({ + id: `${key}-contradiction`, + label: contradictionLabel, + description: contradictionLabel, + kind: "relationship", + status: "supported", + confidence: "medium", + }), + ] + : []; + + const unknowns = [ + makeNode({ + id: `${key}-unknown-a`, + label: "Possible explanation from one side of the situation.", + description: "Possible explanation from one side of the situation.", + kind: "unknown", + status: "unknown", + confidence: "high", + }), + makeNode({ + id: `${key}-unknown-b`, + label: "Possible explanation from another side of the situation.", + description: "Possible explanation from another side of the situation.", + kind: "unknown", + status: "unknown", + confidence: "high", + }), + ]; + + const edges = [ + ...observations.map((node) => + makeEdge({ + id: `${node.id}-supports-summary`, + fromNodeId: node.id, + toNodeId: summary.id, + relationship: "supports", + description: `${node.label} supports the summary.`, + }), + ), + ...unknowns.map((node) => + makeEdge({ + id: `${node.id}-depends-summary`, + fromNodeId: node.id, + toNodeId: summary.id, + relationship: "depends_on", + description: `${node.label} is an unresolved factor for this situation.`, + }), + ), + ]; + + return { + key, + scenario, + expectedComparabilityStatus, + expectsComparisonQuestion, + graph: makeGraph({ + centralStatement: scenario, + nodes: [summary, ...observations, ...contradiction, ...unknowns], + edges, + activeUnknownNodeId: null, + resolvedNodeIds: [], + currentSummary: `Comparability fixture for ${key}`, + }), + }; +} + +export const comparabilityAssessmentFixtures = [ + buildComparabilityFixture({ + key: "revenue-cash", + scenario: + "Revenue increased by 18%, but cash in the bank fell over the same period.", + observationLabels: [ + "Revenue increased by 18%.", + "Cash in the bank decreased over the same period.", + ], + contradictionLabel: + "Contradiction between revenue improvement and lower cash reserves.", + expectedComparabilityStatus: "uncertain", + expectsComparisonQuestion: true, + }), + buildComparabilityFixture({ + key: "complaints-production", + scenario: "Complaints increased. Production increased.", + observationLabels: ["Complaints increased.", "Production increased."], + contradictionLabel: + "Possible contradiction between complaints and production movement.", + expectedComparabilityStatus: "uncertain", + expectsComparisonQuestion: true, + }), + buildComparabilityFixture({ + key: "delivery-cancellations", + scenario: + "Average delivery time decreased by 25%, but order cancellations increased.", + observationLabels: [ + "Average delivery time decreased by 25%.", + "Order cancellations increased.", + ], + contradictionLabel: + "Contradiction between faster delivery and more cancellations.", + expectedComparabilityStatus: "uncertain", + expectsComparisonQuestion: true, + }), + buildComparabilityFixture({ + key: "satisfaction-complaints", + scenario: "Customer satisfaction increased, but complaints increased.", + observationLabels: [ + "Customer satisfaction increased.", + "Complaints increased.", + ], + contradictionLabel: + "Contradiction between satisfaction improvement and more complaints.", + expectedComparabilityStatus: "uncertain", + expectsComparisonQuestion: true, + }), + buildComparabilityFixture({ + key: "temperature-ice", + scenario: "Temperature increased. Ice melted.", + observationLabels: ["Temperature increased.", "Ice melted."], + contradictionLabel: null, + expectedComparabilityStatus: "confirmed", + expectsComparisonQuestion: false, + }), + buildComparabilityFixture({ + key: "sales-same", + scenario: "Sales doubled. Sales doubled.", + observationLabels: ["Sales doubled.", "Sales doubled."], + contradictionLabel: null, + expectedComparabilityStatus: "confirmed", + expectsComparisonQuestion: false, + }), +]; diff --git a/tests/graph/comparability-assessment.test.js b/tests/graph/comparability-assessment.test.js new file mode 100644 index 0000000..0d38918 --- /dev/null +++ b/tests/graph/comparability-assessment.test.js @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; +import { + assessComparability, + formulateTieResolutionQuestion, +} from "@/lib/graph/question-formulator.js"; +import { explainUnknownSelection } from "@/lib/graph/utils.js"; +import { comparabilityAssessmentFixtures } from "@/tests/fixtures/comparability-assessment.js"; + +describe("comparability assessment", () => { + it("generates comparison questions only when comparability is uncertain", () => { + const summary = comparabilityAssessmentFixtures.map((fixture) => { + const assessment = assessComparability(fixture.graph); + const question = formulateTieResolutionQuestion({ graph: fixture.graph }); + const ambiguity = explainUnknownSelection(fixture.graph, []); + + expect(assessment.comparabilityStatus).toBe( + fixture.expectedComparabilityStatus, + ); + expect(question.comparabilityStatus).toBe( + fixture.expectedComparabilityStatus, + ); + + if (fixture.expectsComparisonQuestion) { + expect(question.question.toLowerCase()).toContain("same"); + expect(question.contradictionReasoningAllowed).toBe(false); + } else { + expect(question.question.toLowerCase()).not.toContain( + "same period and at the same scale", + ); + } + + if (fixture.key !== "sales-same") { + expect(ambiguity.status).toBe("ambiguous"); + } + + return { + scenario: fixture.scenario, + comparabilityStatus: assessment.comparabilityStatus, + contradictionReasoningAllowed: assessment.contradictionReasoningAllowed, + question: question.question, + }; + }); + + expect(summary).toMatchInlineSnapshot(` + [ + { + "comparabilityStatus": "uncertain", + "contradictionReasoningAllowed": false, + "question": "Were these figures measured on the same basis and at the same scale?", + "scenario": "Revenue increased by 18%, but cash in the bank fell over the same period.", + }, + { + "comparabilityStatus": "uncertain", + "contradictionReasoningAllowed": false, + "question": "Were these figures measured over the same period and at the same scale?", + "scenario": "Complaints increased. Production increased.", + }, + { + "comparabilityStatus": "uncertain", + "contradictionReasoningAllowed": false, + "question": "Were these figures measured over the same period and at the same scale?", + "scenario": "Average delivery time decreased by 25%, but order cancellations increased.", + }, + { + "comparabilityStatus": "uncertain", + "contradictionReasoningAllowed": false, + "question": "Were these figures measured over the same period and at the same scale?", + "scenario": "Customer satisfaction increased, but complaints increased.", + }, + { + "comparabilityStatus": "confirmed", + "contradictionReasoningAllowed": true, + "question": "What changed during the period that could explain why Temperature increased. Ice melted?", + "scenario": "Temperature increased. Ice melted.", + }, + { + "comparabilityStatus": "confirmed", + "contradictionReasoningAllowed": false, + "question": "What changed during the period that could explain why Sales doubled. Sales doubled?", + "scenario": "Sales doubled. Sales doubled.", + }, + ] + `); + }); +}); diff --git a/tests/graph/orchestrator.test.js b/tests/graph/orchestrator.test.js index e91458c..3693a34 100644 --- a/tests/graph/orchestrator.test.js +++ b/tests/graph/orchestrator.test.js @@ -253,14 +253,18 @@ describe("lib/graph/orchestrator startCase", () => { id: "q_tie_resolution", selectionStatus: "ambiguous", question: - "What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?", + "Were these figures measured on the same basis and at the same scale?", tiedCandidateIds: expect.arrayContaining([expect.any(String)]), + comparabilityStatus: "uncertain", + contradictionReasoningAllowed: false, }); expect(result.diagnostics.unknownSelectionExplanation).toMatchObject({ status: "ambiguous", tieType: "complete_unresolved_tie", selectedNodeId: null, alphabeticalUsedAsReasoning: false, + tieResolutionQuestion: + "Were these figures measured on the same basis and at the same scale?", }); });