confidence-engine/tests/graph/comparability-assessment.test.js

import { describe, expect, it } from "vitest";
import {
  assessComparability,
  classifyObservationRelationship,
  formulateTieResolutionQuestion,
} from "@/lib/graph/question-formulator.js";
import { explainUnknownSelection } from "@/lib/graph/utils.js";
import { comparabilityAssessmentFixtures } from "@/tests/fixtures/comparability-assessment.js";

describe("comparability assessment", () => {
  it("does not mark comparability confirmed when fewer than two supported observations exist", () => {
    const graph = {
      centralStatement: "Only one supported observation exists.",
      nodes: [
        {
          id: "obs-single",
          label: "Revenue increased by 18%.",
          description: "Revenue increased by 18%.",
          kind: "observation",
          status: "supported",
          confidence: "high",
          value: null,
          unit: null,
          evidenceIds: [],
          dependsOn: [],
          affects: [],
          parentId: null,
          childIds: [],
        },
      ],
      edges: [],
      activeUnknownNodeId: null,
      resolvedNodeIds: [],
      currentSummary: "Single observation fixture",
    };

    const assessment = assessComparability(graph);

    expect(assessment.comparabilityStatus).not.toBe("confirmed");
    expect(assessment.contradictionReasoningAllowed).toBe(false);
  });

  it("generates comparison or relationship questions only when warranted", () => {
    const summary = comparabilityAssessmentFixtures.map((fixture) => {
      const assessment = assessComparability(fixture.graph);
      const relationship = classifyObservationRelationship(fixture.graph);
      const question = formulateTieResolutionQuestion({ graph: fixture.graph });
      const ambiguity = explainUnknownSelection(fixture.graph, []);

      expect(assessment.comparabilityStatus).toBe(
        fixture.expectedComparabilityStatus,
      );
      expect(question.comparabilityStatus).toBe(
        fixture.expectedComparabilityStatus,
      );

      if (fixture.expectsComparisonQuestion) {
        expect(question.question.toLowerCase()).toContain("same");
        expect(question.contradictionReasoningAllowed).toBe(false);
      } else {
        expect(question.question?.toLowerCase() || "").not.toContain(
          "same period and at the same scale",
        );
      }

      if (fixture.key !== "sales-same") {
        expect(["ambiguous", "selected"]).toContain(ambiguity.status);
      }

      return {
        scenario: fixture.scenario,
        comparabilityStatus: assessment.comparabilityStatus,
        relationshipStatus: relationship.relationshipStatus,
        relationshipAssessed: relationship.relationshipAssessed,
        contradictionReasoningAllowed: question.contradictionReasoningAllowed,
        question: question.question,
      };
    });

    expect(summary).toEqual([
      {
        scenario:
          "Revenue increased by 18%, but cash in the bank fell over the same period.",
        comparabilityStatus: "uncertain",
        relationshipStatus: "insufficient_information",
        relationshipAssessed: false,
        contradictionReasoningAllowed: false,
        question:
          "Were these figures measured on the same basis and at the same scale?",
      },
      {
        scenario: "Complaints increased. Production increased.",
        comparabilityStatus: "uncertain",
        relationshipStatus: "insufficient_information",
        relationshipAssessed: false,
        contradictionReasoningAllowed: false,
        question:
          "Were these figures measured over the same period and at the same scale?",
      },
      {
        scenario:
          "Average delivery time decreased by 25%, but order cancellations increased.",
        comparabilityStatus: "uncertain",
        relationshipStatus: "insufficient_information",
        relationshipAssessed: false,
        contradictionReasoningAllowed: false,
        question:
          "Were these figures measured over the same period and at the same scale?",
      },
      {
        scenario: "Customer satisfaction increased, but complaints increased.",
        comparabilityStatus: "uncertain",
        relationshipStatus: "insufficient_information",
        relationshipAssessed: false,
        contradictionReasoningAllowed: false,
        question:
          "Were these figures measured over the same period and at the same scale?",
      },
      {
        scenario: "Temperature increased. Ice melted.",
        comparabilityStatus: "confirmed",
        relationshipStatus: "compatible",
        relationshipAssessed: true,
        contradictionReasoningAllowed: false,
        question: null,
      },
      {
        scenario: "Sales doubled. Sales doubled.",
        comparabilityStatus: "confirmed",
        relationshipStatus: "duplicate",
        relationshipAssessed: true,
        contradictionReasoningAllowed: false,
        question: null,
      },
    ]);
  });

  it("defers relationship classification while comparability is uncertain", () => {
    const fixture = comparabilityAssessmentFixtures[0];
    const relationship = classifyObservationRelationship(fixture.graph);

    expect(relationship).toMatchObject({
      relationshipStatus: "insufficient_information",
      relationshipAssessed: false,
      contradictionReasoningAllowed: false,
      questionRequired: true,
    });
    expect(relationship.reasoningStages).toEqual([
      {
        stage: "comparability",
        status: "uncertain",
        outcome:
          "Comparability between the observations is not yet established across period, scale, or measurement basis.",
      },
      {
        stage: "relationship",
        status: "insufficient_information",
        outcome: "not assessed until comparability is established",
      },
    ]);
  });

  it("returns questionRequired=false for fallback (no meaningful relationship structure)", () => {
    // Two non-measurement-like observations with no shared concept and no known direction.
    // They pass comparability (confirmed) but fall through all classification conditions:
    // - not duplicate, no available+unavailable, no revenue/cash pair, no all-known-directions
    // → hits the fallback which now returns questionRequired=false
    const graph = {
      centralStatement:
        "Temperature was measured. Ice was observed.",
      nodes: [
        {
          id: "obs-temperature",
          label: "Temperature was measured.",
          description: "Temperature was measured.",
          kind: "observation",
          status: "supported",
          confidence: "high",
          value: null,
          unit: null,
          evidenceIds: [],
          dependsOn: [],
          affects: [],
          parentId: null,
          childIds: [],
        },
        {
          id: "obs-ice",
          label: "Ice was observed.",
          description: "Ice was observed.",
          kind: "observation",
          status: "supported",
          confidence: "high",
          value: null,
          unit: null,
          evidenceIds: [],
          dependsOn: [],
          affects: [],
          parentId: null,
          childIds: [],
        },
      ],
      edges: [],
      activeUnknownNodeId: null,
      resolvedNodeIds: [],
      currentSummary: "Fallback regression fixture",
    };

    const relationship = classifyObservationRelationship(graph);

    expect(relationship.relationshipStatus).toBe("insufficient_information");
    expect(relationship.questionRequired).toBe(false);
    // Verify reasoning stages exist and have correct status
    const stages = relationship.reasoningStages;
    expect(stages.length).toBe(2);
    expect(stages[0].stage).toBe("comparability");
    expect(stages[0].status).toBe("confirmed");
    expect(stages[1].stage).toBe("relationship");
    expect(stages[1].status).toBe("insufficient_information");

    // questionRequired=false means buildEmergentReasoningUnknown returns early,
    // so no "Explanation for why..." unknown is created — the engine preserves
    // uncertainty rather than fabricating an explanation problem.
  });

  it("allows contradiction reasoning only for genuine contradictions", () => {
    const serviceGraph = {
      centralStatement:
        "The service was reported as available throughout the hour and unavailable throughout the same hour.",
      nodes: [
        {
          id: "service-available",
          label: "The service was available throughout the hour.",
          description: "The service was available throughout the hour.",
          kind: "observation",
          status: "supported",
          confidence: "high",
          value: null,
          unit: null,
          evidenceIds: [],
          dependsOn: [],
          affects: [],
          parentId: null,
          childIds: [],
        },
        {
          id: "service-unavailable",
          label: "The service was unavailable throughout the same hour.",
          description: "The service was unavailable throughout the same hour.",
          kind: "observation",
          status: "supported",
          confidence: "high",
          value: null,
          unit: null,
          evidenceIds: [],
          dependsOn: [],
          affects: [],
          parentId: null,
          childIds: [],
        },
      ],
      edges: [],
      activeUnknownNodeId: null,
      resolvedNodeIds: [],
      currentSummary: "Service contradiction fixture",
    };
    const relationship = classifyObservationRelationship(serviceGraph);

    expect(relationship).toMatchObject({
      relationshipStatus: "contradictory",
      contradictionReasoningAllowed: true,
      questionRequired: true,
    });
  });
});