confidence-engine/tests/llm/provider.test.js

import { describe, expect, it, vi } from "vitest";
import {
  __resetChatSupportForTests,
  createOpenAIReconstructionProvider,
  getProvider,
} from "@/lib/llm/provider.js";

describe("OllamaLlmProvider chat capability detection", () => {
  it("uses the configured model for the chat probe and keeps the chat path", async () => {
    const originalBaseUrl = process.env.OLLAMA_BASE_URL;
    const fetchSpy = vi.fn()
      .mockResolvedValueOnce({ ok: true, text: async () => "" })
      .mockResolvedValueOnce({
        ok: true,
        json: async () => ({ message: { content: "{}" } }),
      });
    const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout");
    vi.stubGlobal("fetch", fetchSpy);
    process.env.OLLAMA_BASE_URL = "http://ollama.test";

    try {
      __resetChatSupportForTests();
      const result = await getProvider().generateReconstruction("prompt", "configured-model");

      expect(JSON.parse(fetchSpy.mock.calls[0][1].body)).toMatchObject({
        model: "configured-model",
        stream: false,
      });
      expect(fetchSpy.mock.calls[0][0]).toBe("http://ollama.test/api/chat");
      expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/chat");
      expect(fetchSpy.mock.calls[1][0]).not.toContain("/api/generate");
      expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), 300000);
      const chatRequest = JSON.parse(fetchSpy.mock.calls[1][1].body);
      expect(chatRequest).toMatchObject({
        model: "configured-model",
        messages: [{ role: "user", content: "prompt" }],
        stream: false,
      });
      expect(chatRequest.format).toBeTypeOf("object");
      expect(chatRequest.format).not.toBe("json");
      const formatText = JSON.stringify(chatRequest.format);
      expect(formatText).toContain("relationship");
      expect(formatText).toContain("evidenceType");
      expect(result).toMatchObject({
        response: {},
        providerApiPath: "/api/chat",
        providerExecution: {
          chatCapabilityDetected: true,
          chatRequestAttempted: true,
          chatRequestSucceeded: true,
          generateRequestAttempted: false,
        },
      });
    } finally {
      setTimeoutSpy.mockRestore();
      vi.unstubAllGlobals();
      if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
      else process.env.OLLAMA_BASE_URL = originalBaseUrl;
    }
  });

  it("reports chat-skipped generate fallback execution", async () => {
    const originalBaseUrl = process.env.OLLAMA_BASE_URL;
    const fetchSpy = vi.fn()
      .mockResolvedValueOnce({ ok: false, status: 501, text: async () => "" })
      .mockResolvedValueOnce({ ok: false, status: 500, text: async () => "failure" });
    vi.stubGlobal("fetch", fetchSpy);
    process.env.OLLAMA_BASE_URL = "http://ollama.test";

    try {
      __resetChatSupportForTests();
      await expect(
        getProvider().generateReconstruction("prompt", "configured-model"),
      ).rejects.toMatchObject({
        providerApiPath: "/api/generate",
        providerExecution: {
          chatCapabilityDetected: false,
          chatRequestAttempted: false,
          chatRequestSucceeded: false,
          generateRequestAttempted: true,
        },
      });
      expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/generate");
    } finally {
      vi.unstubAllGlobals();
      if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
      else process.env.OLLAMA_BASE_URL = originalBaseUrl;
    }
  });

  it("reports chat-attempt-failed generate fallback execution", async () => {
    const originalBaseUrl = process.env.OLLAMA_BASE_URL;
    const fetchSpy = vi.fn()
      .mockResolvedValueOnce({ ok: true, text: async () => "" })
      .mockResolvedValueOnce({ ok: false, text: async () => "" })
      .mockResolvedValueOnce({ ok: false, status: 500, text: async () => "failure" });
    vi.stubGlobal("fetch", fetchSpy);
    process.env.OLLAMA_BASE_URL = "http://ollama.test";

    try {
      __resetChatSupportForTests();
      await expect(
        getProvider().generateReconstruction("prompt", "configured-model"),
      ).rejects.toMatchObject({
        providerApiPath: "/api/generate",
        providerExecution: {
          chatCapabilityDetected: true,
          chatRequestAttempted: true,
          chatRequestSucceeded: false,
          generateRequestAttempted: true,
        },
      });
      expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/chat");
      expect(fetchSpy.mock.calls[2][0]).toBe("http://ollama.test/api/generate");
    } finally {
      vi.unstubAllGlobals();
      if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
      else process.env.OLLAMA_BASE_URL = originalBaseUrl;
    }
  });
});

describe("OpenAI reconstruction provider experiment seam", () => {
  it("uses the Responses API with the canonical strict reconstruction schema", async () => {
    const fetchSpy = vi.fn().mockResolvedValue({
      ok: true,
      json: async () => ({ output_text: '{"reconstruction":"result"}' }),
    });
    const provider = createOpenAIReconstructionProvider({
      apiKey: "test-key",
      fetchImpl: fetchSpy,
    });

    const result = await provider.generateReconstruction(
      "current reconstruction prompt",
    );
    const request = JSON.parse(fetchSpy.mock.calls[0][1].body);
    const schemaText = JSON.stringify(request.text.format.schema);

    expect(fetchSpy).toHaveBeenCalledWith(
      "https://api.openai.com/v1/responses",
      expect.objectContaining({ method: "POST" }),
    );
    expect(request).toMatchObject({
      model: "gpt-5.6-terra",
      input: "current reconstruction prompt",
      text: {
        format: {
          type: "json_schema",
          name: "reconstruction",
          strict: true,
        },
      },
    });
    expect(schemaText).toContain("relationship");
    expect(schemaText).toContain("evidenceType");
    expect(schemaText).toContain("primaryType");
    expect(schemaText).toContain("secondaryTypes");
    expect(schemaText).toContain("confidence");
    expect(schemaText).toContain("importance");
    expect(result).toEqual({
      response: { reconstruction: "result" },
      providerApiPath: "/v1/responses",
    });
  });

  it("surfaces Responses API failures without inventing reconstruction content", async () => {
    const provider = createOpenAIReconstructionProvider({
      apiKey: "test-key",
      fetchImpl: vi.fn().mockResolvedValue({
        ok: false,
        status: 429,
        text: async () => "rate limited",
      }),
    });

    await expect(provider.generateReconstruction("prompt")).rejects.toMatchObject({
      providerApiPath: "/v1/responses",
      message: "OpenAI Responses API returned 429: rate limited",
    });
  });
});