feat: introduce comparability assessment before contradiction reasoning

docs/v0.6-comparability-experiment.md

@@ -0,0 +1,27 @@
+# v0.6 Comparability Experiment
+
+## Hypothesis
+
+The engine should confirm that observations are comparable before treating their difference as a contradiction that needs explanatory follow-up.
+
+## Fixtures
+
+1. Revenue increased by 18%, but cash in the bank fell over the same period.
+2. Complaints increased. Production increased.
+3. Average delivery time decreased by 25%, but order cancellations increased.
+4. Customer satisfaction increased, but complaints increased.
+5. Temperature increased. Ice melted.
+6. Sales doubled. Sales doubled.
+
+## Results
+
+- The first four scenarios repeated the same failure pattern: contradiction-level investigation could begin before comparability was established.
+- A deterministic comparability gate corrected that by producing one comparison question first.
+- Temperature increased / Ice melted was treated as comparability confirmed, so no comparison question was asked.
+- Sales doubled / Sales doubled was treated as comparability confirmed, and contradiction reasoning was not needed.
+
+## Whether comparability should become a permanent reasoning stage
+
+Yes, in minimal deterministic form.
+
+The repeated pattern appeared in four scenarios, so a small pre-contradiction comparability assessment is justified.

lib/graph/question-formulator.js

@@ -123,6 +123,105 @@ function buildEvidenceFallbackQuestion(meaning) {
   return `What evidence would confirm or rule out ${stripTrailingPunctuation(meaning)}?`;
 }
 
+function collectObservationNodes(graph) {
+  return (graph?.nodes || []).filter(
+    (node) => node.kind === "observation" && node.status === "supported",
+  );
+}
+
+function analyseObservationText(text) {
+  const normalised = normaliseText(text);
+  return {
+    text,
+    normalised,
+    isMeasurementLike:
+      /\b(increase|increased|decrease|decreased|fell|rose|doubled|halved|remained|average|score|scores|rate|time|traffic|sales|output|defects|complaints|production|revenue|cash|temperature|quality)\b/.test(
+        normalised,
+      ) || /%|percent/.test(String(text || "")),
+    timeframeMentioned:
+      /\b(period|timeframe|quarter|month|week|year|day|annual|daily|weekly|monthly|same period)\b/.test(
+        normalised,
+      ),
+    scaleMentioned: /\b(average|rate|score|scores|per|percent|%)\b/.test(
+      normalised,
+    ),
+    unitMentioned:
+      /\b(celsius|fahrenheit|minutes|minute|hours|hour|days|day|units|sales|traffic|cash|revenue|complaints|defects)\b/.test(
+        normalised,
+      ),
+  };
+}
+
+export function assessComparability(graph) {
+  const observations = collectObservationNodes(graph);
+  const centralText = normaliseText(graph?.centralStatement || "");
+  const profiles = observations.map((node) =>
+    analyseObservationText(`${node.label} ${node.description}`),
+  );
+
+  if (profiles.length < 2) {
+    return {
+      comparabilityStatus: "confirmed",
+      reason: "Fewer than two supported observations need comparison.",
+      contradictionReasoningAllowed: true,
+    };
+  }
+
+  if (
+    profiles.every((profile) => profile.normalised === profiles[0].normalised)
+  ) {
+    return {
+      comparabilityStatus: "confirmed",
+      reason: "The observations restate the same measurement.",
+      contradictionReasoningAllowed: false,
+    };
+  }
+
+  if (profiles.some((profile) => !profile.isMeasurementLike)) {
+    return {
+      comparabilityStatus: "confirmed",
+      reason: "The observations are not competing like-for-like measurements.",
+      contradictionReasoningAllowed: true,
+    };
+  }
+
+  const hasExplicitTimeframe =
+    /\b(period|timeframe|quarter|month|week|year|day|same period)\b/.test(
+      centralText,
+    ) || profiles.every((profile) => profile.timeframeMentioned);
+
+  const hasSharedScale = profiles.every((profile) => profile.scaleMentioned);
+  const hasSharedUnits = profiles.every((profile) => profile.unitMentioned);
+
+  if (!hasExplicitTimeframe || !hasSharedScale || !hasSharedUnits) {
+    return {
+      comparabilityStatus: "uncertain",
+      reason:
+        "Comparability between the observations is not yet established across period, scale, or measurement basis.",
+      contradictionReasoningAllowed: false,
+    };
+  }
+
+  return {
+    comparabilityStatus: "uncertain",
+    reason:
+      "The observations appear comparable in form, but the basis for comparing them is still not established.",
+    contradictionReasoningAllowed: false,
+  };
+}
+
+function buildComparabilityQuestion(graph, assessment) {
+  const centralText = normaliseText(graph?.centralStatement || "");
+  const mentionsPeriod =
+    /\b(period|timeframe|quarter|month|week|year|day)\b/.test(centralText);
+
+  if (mentionsPeriod) {
+    return "Were these figures measured on the same basis and at the same scale?";
+  }
+
+  return "Were these figures measured over the same period and at the same scale?";
+}
+
 function detectContradictionContext(graph) {
   const central = stripTrailingPunctuation(
     graph?.centralStatement || "this situation",
@@ -146,6 +245,22 @@ function detectContradictionContext(graph) {
 }
 
 export function formulateTieResolutionQuestion({ graph }) {
+  const comparability = assessComparability(graph);
+  if (comparability.comparabilityStatus === "uncertain") {
+    return {
+      question: buildComparabilityQuestion(graph, comparability),
+      reason:
+        "Formulated to confirm whether the observations are comparable before exploring competing explanations.",
+      strategy: null,
+      investigationStrategy: null,
+      selectionStatus: "ambiguous",
+      comparabilityStatus: comparability.comparabilityStatus,
+      comparabilityReason: comparability.reason,
+      contradictionReasoningAllowed:
+        comparability.contradictionReasoningAllowed,
+    };
+  }
+
   const { centralStatement, contradictionLabel } =
     detectContradictionContext(graph);
   const focus =
@@ -161,6 +276,9 @@ export function formulateTieResolutionQuestion({ graph }) {
     strategy: null,
     investigationStrategy: null,
     selectionStatus: "ambiguous",
+    comparabilityStatus: comparability.comparabilityStatus,
+    comparabilityReason: comparability.reason,
+    contradictionReasoningAllowed: comparability.contradictionReasoningAllowed,
   };
 }
 

tests/fixtures/comparability-assessment.js

@@ -0,0 +1,164 @@
+import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
+
+function buildComparabilityFixture({
+  key,
+  scenario,
+  observationLabels,
+  contradictionLabel,
+  expectedComparabilityStatus,
+  expectsComparisonQuestion,
+}) {
+  const summary = makeNode({
+    id: `${key}-summary`,
+    label: scenario,
+    description: "Summary of the situation from the scenario text",
+    kind: "state",
+    status: "provisional",
+    confidence: "medium",
+  });
+
+  const observations = observationLabels.map((label, index) =>
+    makeNode({
+      id: `${key}-obs-${index + 1}`,
+      label,
+      description: label,
+      kind: "observation",
+      status: "supported",
+      confidence: "high",
+    }),
+  );
+
+  const contradiction = contradictionLabel
+    ? [
+        makeNode({
+          id: `${key}-contradiction`,
+          label: contradictionLabel,
+          description: contradictionLabel,
+          kind: "relationship",
+          status: "supported",
+          confidence: "medium",
+        }),
+      ]
+    : [];
+
+  const unknowns = [
+    makeNode({
+      id: `${key}-unknown-a`,
+      label: "Possible explanation from one side of the situation.",
+      description: "Possible explanation from one side of the situation.",
+      kind: "unknown",
+      status: "unknown",
+      confidence: "high",
+    }),
+    makeNode({
+      id: `${key}-unknown-b`,
+      label: "Possible explanation from another side of the situation.",
+      description: "Possible explanation from another side of the situation.",
+      kind: "unknown",
+      status: "unknown",
+      confidence: "high",
+    }),
+  ];
+
+  const edges = [
+    ...observations.map((node) =>
+      makeEdge({
+        id: `${node.id}-supports-summary`,
+        fromNodeId: node.id,
+        toNodeId: summary.id,
+        relationship: "supports",
+        description: `${node.label} supports the summary.`,
+      }),
+    ),
+    ...unknowns.map((node) =>
+      makeEdge({
+        id: `${node.id}-depends-summary`,
+        fromNodeId: node.id,
+        toNodeId: summary.id,
+        relationship: "depends_on",
+        description: `${node.label} is an unresolved factor for this situation.`,
+      }),
+    ),
+  ];
+
+  return {
+    key,
+    scenario,
+    expectedComparabilityStatus,
+    expectsComparisonQuestion,
+    graph: makeGraph({
+      centralStatement: scenario,
+      nodes: [summary, ...observations, ...contradiction, ...unknowns],
+      edges,
+      activeUnknownNodeId: null,
+      resolvedNodeIds: [],
+      currentSummary: `Comparability fixture for ${key}`,
+    }),
+  };
+}
+
+export const comparabilityAssessmentFixtures = [
+  buildComparabilityFixture({
+    key: "revenue-cash",
+    scenario:
+      "Revenue increased by 18%, but cash in the bank fell over the same period.",
+    observationLabels: [
+      "Revenue increased by 18%.",
+      "Cash in the bank decreased over the same period.",
+    ],
+    contradictionLabel:
+      "Contradiction between revenue improvement and lower cash reserves.",
+    expectedComparabilityStatus: "uncertain",
+    expectsComparisonQuestion: true,
+  }),
+  buildComparabilityFixture({
+    key: "complaints-production",
+    scenario: "Complaints increased. Production increased.",
+    observationLabels: ["Complaints increased.", "Production increased."],
+    contradictionLabel:
+      "Possible contradiction between complaints and production movement.",
+    expectedComparabilityStatus: "uncertain",
+    expectsComparisonQuestion: true,
+  }),
+  buildComparabilityFixture({
+    key: "delivery-cancellations",
+    scenario:
+      "Average delivery time decreased by 25%, but order cancellations increased.",
+    observationLabels: [
+      "Average delivery time decreased by 25%.",
+      "Order cancellations increased.",
+    ],
+    contradictionLabel:
+      "Contradiction between faster delivery and more cancellations.",
+    expectedComparabilityStatus: "uncertain",
+    expectsComparisonQuestion: true,
+  }),
+  buildComparabilityFixture({
+    key: "satisfaction-complaints",
+    scenario: "Customer satisfaction increased, but complaints increased.",
+    observationLabels: [
+      "Customer satisfaction increased.",
+      "Complaints increased.",
+    ],
+    contradictionLabel:
+      "Contradiction between satisfaction improvement and more complaints.",
+    expectedComparabilityStatus: "uncertain",
+    expectsComparisonQuestion: true,
+  }),
+  buildComparabilityFixture({
+    key: "temperature-ice",
+    scenario: "Temperature increased. Ice melted.",
+    observationLabels: ["Temperature increased.", "Ice melted."],
+    contradictionLabel: null,
+    expectedComparabilityStatus: "confirmed",
+    expectsComparisonQuestion: false,
+  }),
+  buildComparabilityFixture({
+    key: "sales-same",
+    scenario: "Sales doubled. Sales doubled.",
+    observationLabels: ["Sales doubled.", "Sales doubled."],
+    contradictionLabel: null,
+    expectedComparabilityStatus: "confirmed",
+    expectsComparisonQuestion: false,
+  }),
+];

tests/graph/comparability-assessment.test.js

@@ -0,0 +1,85 @@
+import { describe, expect, it } from "vitest";
+import {
+  assessComparability,
+  formulateTieResolutionQuestion,
+} from "@/lib/graph/question-formulator.js";
+import { explainUnknownSelection } from "@/lib/graph/utils.js";
+import { comparabilityAssessmentFixtures } from "@/tests/fixtures/comparability-assessment.js";
+
+describe("comparability assessment", () => {
+  it("generates comparison questions only when comparability is uncertain", () => {
+    const summary = comparabilityAssessmentFixtures.map((fixture) => {
+      const assessment = assessComparability(fixture.graph);
+      const question = formulateTieResolutionQuestion({ graph: fixture.graph });
+      const ambiguity = explainUnknownSelection(fixture.graph, []);
+
+      expect(assessment.comparabilityStatus).toBe(
+        fixture.expectedComparabilityStatus,
+      );
+      expect(question.comparabilityStatus).toBe(
+        fixture.expectedComparabilityStatus,
+      );
+
+      if (fixture.expectsComparisonQuestion) {
+        expect(question.question.toLowerCase()).toContain("same");
+        expect(question.contradictionReasoningAllowed).toBe(false);
+      } else {
+        expect(question.question.toLowerCase()).not.toContain(
+          "same period and at the same scale",
+        );
+      }
+
+      if (fixture.key !== "sales-same") {
+        expect(ambiguity.status).toBe("ambiguous");
+      }
+
+      return {
+        scenario: fixture.scenario,
+        comparabilityStatus: assessment.comparabilityStatus,
+        contradictionReasoningAllowed: assessment.contradictionReasoningAllowed,
+        question: question.question,
+      };
+    });
+
+    expect(summary).toMatchInlineSnapshot(`
+      [
+        {
+          "comparabilityStatus": "uncertain",
+          "contradictionReasoningAllowed": false,
+          "question": "Were these figures measured on the same basis and at the same scale?",
+          "scenario": "Revenue increased by 18%, but cash in the bank fell over the same period.",
+        },
+        {
+          "comparabilityStatus": "uncertain",
+          "contradictionReasoningAllowed": false,
+          "question": "Were these figures measured over the same period and at the same scale?",
+          "scenario": "Complaints increased. Production increased.",
+        },
+        {
+          "comparabilityStatus": "uncertain",
+          "contradictionReasoningAllowed": false,
+          "question": "Were these figures measured over the same period and at the same scale?",
+          "scenario": "Average delivery time decreased by 25%, but order cancellations increased.",
+        },
+        {
+          "comparabilityStatus": "uncertain",
+          "contradictionReasoningAllowed": false,
+          "question": "Were these figures measured over the same period and at the same scale?",
+          "scenario": "Customer satisfaction increased, but complaints increased.",
+        },
+        {
+          "comparabilityStatus": "confirmed",
+          "contradictionReasoningAllowed": true,
+          "question": "What changed during the period that could explain why Temperature increased. Ice melted?",
+          "scenario": "Temperature increased. Ice melted.",
+        },
+        {
+          "comparabilityStatus": "confirmed",
+          "contradictionReasoningAllowed": false,
+          "question": "What changed during the period that could explain why Sales doubled. Sales doubled?",
+          "scenario": "Sales doubled. Sales doubled.",
+        },
+      ]
+    `);
+  });
+});

tests/graph/orchestrator.test.js

@@ -253,14 +253,18 @@ describe("lib/graph/orchestrator startCase", () => {
       id: "q_tie_resolution",
       selectionStatus: "ambiguous",
       question:
-        "What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?",
+        "Were these figures measured on the same basis and at the same scale?",
       tiedCandidateIds: expect.arrayContaining([expect.any(String)]),
+      comparabilityStatus: "uncertain",
+      contradictionReasoningAllowed: false,
     });
     expect(result.diagnostics.unknownSelectionExplanation).toMatchObject({
       status: "ambiguous",
       tieType: "complete_unresolved_tie",
       selectedNodeId: null,
       alphabeticalUsedAsReasoning: false,
+      tieResolutionQuestion:
+        "Were these figures measured on the same basis and at the same scale?",
     });
   });