confidence-engine/tests/graph/comparability-assessment.test.js

import { describe, expect, it } from "vitest";
import {
  assessComparability,
  classifyObservationRelationship,
  formulateTieResolutionQuestion,
} from "@/lib/graph/question-formulator.js";
import { explainUnknownSelection } from "@/lib/graph/utils.js";
import { comparabilityAssessmentFixtures } from "@/tests/fixtures/comparability-assessment.js";

describe("comparability assessment", () => {
  it("generates comparison or relationship questions only when warranted", () => {
    const summary = comparabilityAssessmentFixtures.map((fixture) => {
      const assessment = assessComparability(fixture.graph);
      const relationship = classifyObservationRelationship(fixture.graph);
      const question = formulateTieResolutionQuestion({ graph: fixture.graph });
      const ambiguity = explainUnknownSelection(fixture.graph, []);

      expect(assessment.comparabilityStatus).toBe(
        fixture.expectedComparabilityStatus,
      );
      expect(question.comparabilityStatus).toBe(
        fixture.expectedComparabilityStatus,
      );

      if (fixture.expectsComparisonQuestion) {
        expect(question.question.toLowerCase()).toContain("same");
        expect(question.contradictionReasoningAllowed).toBe(false);
      } else {
        expect(question.question?.toLowerCase() || "").not.toContain(
          "same period and at the same scale",
        );
      }

      if (fixture.key !== "sales-same") {
        expect(ambiguity.status).toBe("ambiguous");
      }

      return {
        scenario: fixture.scenario,
        comparabilityStatus: assessment.comparabilityStatus,
        relationshipStatus: relationship.relationshipStatus,
        relationshipAssessed: relationship.relationshipAssessed,
        contradictionReasoningAllowed: question.contradictionReasoningAllowed,
        question: question.question,
      };
    });

    expect(summary).toEqual([
      {
        scenario:
          "Revenue increased by 18%, but cash in the bank fell over the same period.",
        comparabilityStatus: "uncertain",
        relationshipStatus: "insufficient_information",
        relationshipAssessed: false,
        contradictionReasoningAllowed: false,
        question:
          "Were these figures measured on the same basis and at the same scale?",
      },
      {
        scenario: "Complaints increased. Production increased.",
        comparabilityStatus: "uncertain",
        relationshipStatus: "insufficient_information",
        relationshipAssessed: false,
        contradictionReasoningAllowed: false,
        question:
          "Were these figures measured over the same period and at the same scale?",
      },
      {
        scenario:
          "Average delivery time decreased by 25%, but order cancellations increased.",
        comparabilityStatus: "uncertain",
        relationshipStatus: "insufficient_information",
        relationshipAssessed: false,
        contradictionReasoningAllowed: false,
        question:
          "Were these figures measured over the same period and at the same scale?",
      },
      {
        scenario: "Customer satisfaction increased, but complaints increased.",
        comparabilityStatus: "uncertain",
        relationshipStatus: "insufficient_information",
        relationshipAssessed: false,
        contradictionReasoningAllowed: false,
        question:
          "Were these figures measured over the same period and at the same scale?",
      },
      {
        scenario: "Temperature increased. Ice melted.",
        comparabilityStatus: "confirmed",
        relationshipStatus: "compatible",
        relationshipAssessed: true,
        contradictionReasoningAllowed: false,
        question: null,
      },
      {
        scenario: "Sales doubled. Sales doubled.",
        comparabilityStatus: "confirmed",
        relationshipStatus: "duplicate",
        relationshipAssessed: true,
        contradictionReasoningAllowed: false,
        question: null,
      },
    ]);
  });

  it("defers relationship classification while comparability is uncertain", () => {
    const fixture = comparabilityAssessmentFixtures[0];
    const relationship = classifyObservationRelationship(fixture.graph);

    expect(relationship).toMatchObject({
      relationshipStatus: "insufficient_information",
      relationshipAssessed: false,
      contradictionReasoningAllowed: false,
      questionRequired: true,
    });
    expect(relationship.reasoningStages).toEqual([
      {
        stage: "comparability",
        status: "uncertain",
        outcome:
          "Comparability between the observations is not yet established across period, scale, or measurement basis.",
      },
      {
        stage: "relationship",
        status: "insufficient_information",
        outcome: "not assessed until comparability is established",
      },
    ]);
  });

  it("allows contradiction reasoning only for genuine contradictions", () => {
    const serviceGraph = {
      centralStatement:
        "The service was reported as available throughout the hour and unavailable throughout the same hour.",
      nodes: [
        {
          id: "service-available",
          label: "The service was available throughout the hour.",
          description: "The service was available throughout the hour.",
          kind: "observation",
          status: "supported",
          confidence: "high",
          value: null,
          unit: null,
          evidenceIds: [],
          dependsOn: [],
          affects: [],
          parentId: null,
          childIds: [],
        },
        {
          id: "service-unavailable",
          label: "The service was unavailable throughout the same hour.",
          description: "The service was unavailable throughout the same hour.",
          kind: "observation",
          status: "supported",
          confidence: "high",
          value: null,
          unit: null,
          evidenceIds: [],
          dependsOn: [],
          affects: [],
          parentId: null,
          childIds: [],
        },
      ],
      edges: [],
      activeUnknownNodeId: null,
      resolvedNodeIds: [],
      currentSummary: "Service contradiction fixture",
    };
    const relationship = classifyObservationRelationship(serviceGraph);

    expect(relationship).toMatchObject({
      relationshipStatus: "contradictory",
      contradictionReasoningAllowed: true,
      questionRequired: true,
    });
  });
});