confidence-engine/tests/reconstruction/compatibility.test.js

import { beforeEach, describe, expect, it, vi } from "vitest";
import { normaliseAnalysisResponse } from "@/lib/reconstruction/compatibility.js";

const mockGenerateReconstruction = vi.fn();

vi.mock("@/lib/config.js", () => ({
  getConfig: () => ({
    ok: true,
    config: {
      OLLAMA_BASE_URL: "http://example.test",
      OLLAMA_MODEL: "test-model",
    },
  }),
}));

vi.mock("@/lib/llm/provider.js", () => ({
  getProvider: () => ({
    generateReconstruction: (...args) => mockGenerateReconstruction(...args),
  }),
}));

vi.mock("@/lib/reconstruction/prompt.js", () => ({
  buildPrompt: async () => ({ prompt: "prompt", version: "v0.3" }),
  PROMPT_VERSIONS: ["v0.1", "v0.2", "v0.3"],
  DEFAULT_PROMPT_VERSION: "v0.3",
}));

describe("normaliseAnalysisResponse", () => {
  beforeEach(() => {
    vi.resetModules();
    vi.clearAllMocks();
  });

  it("leaves already-valid responses unchanged", () => {
    const input = {
      evidence: [
        {
          id: "ev1",
          description: "x",
          evidenceType: "reported_statement",
          confidence: "medium",
          importance: "important",
          source: "report",
        },
      ],
    };

    const result = normaliseAnalysisResponse(input);

    expect(result.normalised).toEqual(input);
    expect(result.changesApplied).toEqual([]);
  });

  it("normalises null evidence source deterministically", () => {
    const input = {
      evidence: [
        {
          id: "ev1",
          description: "x",
          evidenceType: "reported_statement",
          confidence: "medium",
          importance: "important",
          source: null,
        },
      ],
    };

    const result = normaliseAnalysisResponse(input);

    expect(result.normalised.evidence[0]).not.toHaveProperty("source");
    expect(result.changesApplied).toHaveLength(1);
  });

  it("normalises reported_claim evidenceType to reported_statement", () => {
    const input = {
      evidence: [
        {
          id: "ev1",
          description: "x",
          evidenceType: "reported_claim",
          confidence: "medium",
          importance: "important",
          source: "report",
        },
      ],
    };

    const result = normaliseAnalysisResponse(input);

    expect(result.normalised.evidence[0].evidenceType).toBe(
      "reported_statement",
    );
    expect(result.changesApplied).toEqual([
      {
        path: ["evidence", 0, "evidenceType"],
        change: "Converted reported_claim to reported_statement",
      },
    ]);
  });

  it("does not invent a next question", () => {
    const input = { evidence: [] };
    const result = normaliseAnalysisResponse(input);
    expect(result.normalised.nextQuestion).toBeUndefined();
  });

  it("does not repair missing reasoning content", () => {
    const input = { evidence: [{ source: null }] };
    const result = normaliseAnalysisResponse(input);
    expect(result.normalised.reconstruction).toBeUndefined();
  });
});

describe("analyseScenario compatibility", () => {
  it("preserves an attempted provider API path on provider failure", async () => {
    const providerError = new Error("Provider failed");
    providerError.providerApiPath = "/api/chat";
    providerError.providerExecution = {
      chatCapabilityDetected: true,
      chatRequestAttempted: true,
      chatRequestSucceeded: false,
      generateRequestAttempted: true,
    };
    mockGenerateReconstruction.mockRejectedValue(providerError);

    const { analyseScenario } = await import("@/lib/analysis.js");
    const result = await analyseScenario("Scenario text", {
      promptVersion: "v0.3",
    });

    expect(result).toMatchObject({
      success: false,
      providerApiPath: "/api/chat",
      providerExecution: providerError.providerExecution,
    });
  });

  it("succeeds when the only mismatch is null evidence source", async () => {
    mockGenerateReconstruction.mockResolvedValue({
      inputClassification: {
        primaryType: "unexplained_change",
        secondaryTypes: [],
        reasoningModes: ["validate_measurement"],
        classificationReason: "reason",
        confidence: "medium",
      },
      reconstruction: {
        summary: "summary",
        actors: [],
        systemsOrObjects: [],
        expectedStates: [],
        observedStates: [],
        differences: [],
        knownTransitions: [],
        unexplainedTransitions: [],
        contradictions: [],
        importantUnknowns: [],
        plausibleInterpretations: [],
      },
      evidence: [
        {
          id: "ev1",
          description: "desc",
          evidenceType: "reported_statement",
          source: null,
          attribution: null,
          confidence: "medium",
          importance: "important",
        },
      ],
      nextQuestion: {
        id: "q1",
        question: "What denominator?",
        targets: ["observedStates"],
        reason: "reason",
        expectedInformationValue: "high",
        reasoningMode: "validate_measurement",
      },
    });

    const { analyseScenario } = await import("@/lib/analysis.js");
    const result = await analyseScenario("Scenario text", {
      promptVersion: "v0.3",
    });

    expect(result.success).toBe(true);
    expect(result.compatibilityApplied).toBe(true);
    expect(result.compatibilityChanges).toHaveLength(1);
    expect(result.evidence[0]).not.toHaveProperty("source");
    expect(result.nextQuestion.question).toBe("What denominator?");
  });

  it("still fails when required reasoning content is missing", async () => {
    mockGenerateReconstruction.mockResolvedValue({
      evidence: [
        {
          id: "ev1",
          description: "desc",
          evidenceType: "reported_statement",
          source: null,
          attribution: null,
          confidence: "medium",
          importance: "important",
        },
      ],
    });

    const { analyseScenario } = await import("@/lib/analysis.js");
    const result = await analyseScenario("Scenario text", {
      promptVersion: "v0.3",
    });

    expect(result.success).toBe(false);
    expect(result.compatibilityApplied).toBe(true);
    expect(result.nextQuestion).toBeUndefined();
  });

  it("preserves nested validation issues and complete raw output on reconstruction failure", async () => {
    mockGenerateReconstruction.mockResolvedValue({
      providerApiPath: "/api/chat",
      response: {
      inputClassification: {
        primaryType: "unexplained_change",
        secondaryTypes: [],
        reasoningModes: [],
        classificationReason: "reason",
        confidence: "medium",
      },
      reconstruction: {
        summary: "summary",
        actors: [],
        systemsOrObjects: [],
        expectedStates: [],
        observedStates: [{ id: "obs-1", confidence: "high" }],
        differences: [],
        knownTransitions: [],
        unexplainedTransitions: [],
        contradictions: [],
        importantUnknowns: [],
        plausibleInterpretations: [],
        padding: "x".repeat(2500),
      },
      evidence: [],
      nextQuestion: {
        id: "q1",
        question: "What changed?",
        targets: [],
        reason: "reason",
        expectedInformationValue: "high",
      },
      },
    });

    const { analyseScenario } = await import("@/lib/analysis.js");
    const result = await analyseScenario("Scenario text", {
      promptVersion: "v0.3",
    });

    expect(result.success).toBe(false);
    expect(result.rawResponse.length).toBeGreaterThan(2000);
    expect(result.validationIssues).toContainEqual(expect.objectContaining({
      path: ["reconstruction", "observedStates", 0, "description"],
      code: "invalid_type",
      expected: "string",
    }));
    expect(result.providerApiPath).toBe("/api/chat");
    expect(result.errors).toContainEqual(expect.stringMatching(/^reconstruction:/));
  });

  it("succeeds when reported_claim is the only evidenceType mismatch", async () => {
    mockGenerateReconstruction.mockResolvedValue({
      inputClassification: {
        primaryType: "unexplained_change",
        secondaryTypes: [],
        reasoningModes: ["validate_measurement"],
        classificationReason: "reason",
        confidence: "medium",
      },
      reconstruction: {
        summary: "summary",
        actors: [],
        systemsOrObjects: [],
        expectedStates: [],
        observedStates: [],
        differences: [],
        knownTransitions: [],
        unexplainedTransitions: [],
        contradictions: [],
        importantUnknowns: [],
        plausibleInterpretations: [],
      },
      evidence: [
        {
          id: "ev1",
          description: "desc",
          evidenceType: "reported_claim",
          attribution: null,
          confidence: "medium",
          importance: "important",
        },
      ],
      nextQuestion: {
        id: "q1",
        question: "What denominator?",
        targets: ["observedStates"],
        reason: "reason",
        expectedInformationValue: "high",
        reasoningMode: "validate_measurement",
      },
    });

    const { analyseScenario } = await import("@/lib/analysis.js");
    const result = await analyseScenario("Scenario text", {
      promptVersion: "v0.3",
    });

    expect(result.success).toBe(true);
    expect(result.compatibilityApplied).toBe(true);
    expect(result.compatibilityChanges).toContainEqual({
      path: ["evidence", 0, "evidenceType"],
      change: "Converted reported_claim to reported_statement",
    });
    expect(result.evidence[0].evidenceType).toBe("reported_statement");
  });

  it("malformed JSON still fails", async () => {
    mockGenerateReconstruction.mockResolvedValue("{not valid json");

    const { analyseScenario } = await import("@/lib/analysis.js");
    const result = await analyseScenario("Scenario text", {
      promptVersion: "v0.3",
    });

    expect(result.success).toBe(false);
    expect(result.compatibilityApplied).toBe(false);
  });
});