257 lines
9.6 KiB
JavaScript
257 lines
9.6 KiB
JavaScript
import { describe, expect, it, vi } from "vitest";
|
|
import {
|
|
__resetChatSupportForTests,
|
|
createOpenAIStrictSchema,
|
|
createOpenAIReconstructionProvider,
|
|
getProvider,
|
|
normaliseOpenAITransportResponse,
|
|
} from "@/lib/llm/provider.js";
|
|
import { reconstructionV2Schema } from "@/lib/reconstruction/schema.js";
|
|
import { z } from "zod";
|
|
|
|
describe("OllamaLlmProvider chat capability detection", () => {
|
|
it("uses the configured model for the chat probe and keeps the chat path", async () => {
|
|
const originalBaseUrl = process.env.OLLAMA_BASE_URL;
|
|
const fetchSpy = vi.fn()
|
|
.mockResolvedValueOnce({ ok: true, text: async () => "" })
|
|
.mockResolvedValueOnce({
|
|
ok: true,
|
|
json: async () => ({ message: { content: "{}" } }),
|
|
});
|
|
const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout");
|
|
vi.stubGlobal("fetch", fetchSpy);
|
|
process.env.OLLAMA_BASE_URL = "http://ollama.test";
|
|
|
|
try {
|
|
__resetChatSupportForTests();
|
|
const result = await getProvider().generateReconstruction("prompt", "configured-model");
|
|
|
|
expect(JSON.parse(fetchSpy.mock.calls[0][1].body)).toMatchObject({
|
|
model: "configured-model",
|
|
stream: false,
|
|
});
|
|
expect(fetchSpy.mock.calls[0][0]).toBe("http://ollama.test/api/chat");
|
|
expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/chat");
|
|
expect(fetchSpy.mock.calls[1][0]).not.toContain("/api/generate");
|
|
expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), 300000);
|
|
const chatRequest = JSON.parse(fetchSpy.mock.calls[1][1].body);
|
|
expect(chatRequest).toMatchObject({
|
|
model: "configured-model",
|
|
messages: [{ role: "user", content: "prompt" }],
|
|
stream: false,
|
|
});
|
|
expect(chatRequest.format).toBeTypeOf("object");
|
|
expect(chatRequest.format).not.toBe("json");
|
|
const formatText = JSON.stringify(chatRequest.format);
|
|
expect(formatText).toContain("relationship");
|
|
expect(formatText).toContain("evidenceType");
|
|
expect(result).toMatchObject({
|
|
response: {},
|
|
providerApiPath: "/api/chat",
|
|
providerExecution: {
|
|
chatCapabilityDetected: true,
|
|
chatRequestAttempted: true,
|
|
chatRequestSucceeded: true,
|
|
generateRequestAttempted: false,
|
|
},
|
|
});
|
|
} finally {
|
|
setTimeoutSpy.mockRestore();
|
|
vi.unstubAllGlobals();
|
|
if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
|
|
else process.env.OLLAMA_BASE_URL = originalBaseUrl;
|
|
}
|
|
});
|
|
|
|
it("reports chat-skipped generate fallback execution", async () => {
|
|
const originalBaseUrl = process.env.OLLAMA_BASE_URL;
|
|
const fetchSpy = vi.fn()
|
|
.mockResolvedValueOnce({ ok: false, status: 501, text: async () => "" })
|
|
.mockResolvedValueOnce({ ok: false, status: 500, text: async () => "failure" });
|
|
vi.stubGlobal("fetch", fetchSpy);
|
|
process.env.OLLAMA_BASE_URL = "http://ollama.test";
|
|
|
|
try {
|
|
__resetChatSupportForTests();
|
|
await expect(
|
|
getProvider().generateReconstruction("prompt", "configured-model"),
|
|
).rejects.toMatchObject({
|
|
providerApiPath: "/api/generate",
|
|
providerExecution: {
|
|
chatCapabilityDetected: false,
|
|
chatRequestAttempted: false,
|
|
chatRequestSucceeded: false,
|
|
generateRequestAttempted: true,
|
|
},
|
|
});
|
|
expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/generate");
|
|
} finally {
|
|
vi.unstubAllGlobals();
|
|
if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
|
|
else process.env.OLLAMA_BASE_URL = originalBaseUrl;
|
|
}
|
|
});
|
|
|
|
it("reports chat-attempt-failed generate fallback execution", async () => {
|
|
const originalBaseUrl = process.env.OLLAMA_BASE_URL;
|
|
const fetchSpy = vi.fn()
|
|
.mockResolvedValueOnce({ ok: true, text: async () => "" })
|
|
.mockResolvedValueOnce({ ok: false, text: async () => "" })
|
|
.mockResolvedValueOnce({ ok: false, status: 500, text: async () => "failure" });
|
|
vi.stubGlobal("fetch", fetchSpy);
|
|
process.env.OLLAMA_BASE_URL = "http://ollama.test";
|
|
|
|
try {
|
|
__resetChatSupportForTests();
|
|
await expect(
|
|
getProvider().generateReconstruction("prompt", "configured-model"),
|
|
).rejects.toMatchObject({
|
|
providerApiPath: "/api/generate",
|
|
providerExecution: {
|
|
chatCapabilityDetected: true,
|
|
chatRequestAttempted: true,
|
|
chatRequestSucceeded: false,
|
|
generateRequestAttempted: true,
|
|
},
|
|
});
|
|
expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/chat");
|
|
expect(fetchSpy.mock.calls[2][0]).toBe("http://ollama.test/api/generate");
|
|
} finally {
|
|
vi.unstubAllGlobals();
|
|
if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
|
|
else process.env.OLLAMA_BASE_URL = originalBaseUrl;
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("OpenAI reconstruction provider experiment seam", () => {
|
|
it("projects canonical optional fields as required but nullable", () => {
|
|
const nativeSchema = z.toJSONSchema(reconstructionV2Schema);
|
|
const projectedSchema = createOpenAIStrictSchema(
|
|
nativeSchema,
|
|
reconstructionV2Schema,
|
|
);
|
|
|
|
function resolveLocalRef(schema, root = projectedSchema) {
|
|
if (!schema.$ref) return schema;
|
|
return schema.$ref
|
|
.replace(/^#\//, "")
|
|
.split("/")
|
|
.reduce((value, key) => value?.[key], root);
|
|
}
|
|
|
|
function acceptsNull(schema, root = projectedSchema) {
|
|
const resolved = resolveLocalRef(schema, root);
|
|
return (
|
|
resolved?.type === "null" ||
|
|
(Array.isArray(resolved?.type) && resolved.type.includes("null")) ||
|
|
[...(resolved?.anyOf ?? []), ...(resolved?.oneOf ?? [])].some((branch) =>
|
|
acceptsNull(branch, root),
|
|
)
|
|
);
|
|
}
|
|
|
|
function assertAllPropertiesRequired(schema, root = projectedSchema) {
|
|
const resolved = resolveLocalRef(schema, root);
|
|
if (resolved?.properties) {
|
|
expect(resolved.required).toEqual(Object.keys(resolved.properties));
|
|
Object.values(resolved.properties).forEach((property) => assertAllPropertiesRequired(property, root));
|
|
}
|
|
(resolved?.anyOf ?? []).forEach((branch) => assertAllPropertiesRequired(branch, root));
|
|
(resolved?.oneOf ?? []).forEach((branch) => assertAllPropertiesRequired(branch, root));
|
|
if (resolved?.items) assertAllPropertiesRequired(resolved.items, root);
|
|
}
|
|
|
|
assertAllPropertiesRequired(projectedSchema);
|
|
const transition = projectedSchema.properties.reconstruction.properties.unexplainedTransitions.items;
|
|
expect(transition.required).toContain("entity");
|
|
expect(acceptsNull(transition.properties.entity)).toBe(true);
|
|
const inputClassification = resolveLocalRef(
|
|
projectedSchema.properties.inputClassification,
|
|
);
|
|
expect(inputClassification.required).toContain("secondaryTypes");
|
|
expect(acceptsNull(inputClassification.properties.secondaryTypes)).toBe(true);
|
|
expect(acceptsNull(transition.properties.id)).toBe(false);
|
|
});
|
|
|
|
it("removes only optional transport null placeholders", () => {
|
|
const schema = z.object({
|
|
optionalText: z.string().optional(),
|
|
nullableText: z.string().nullable(),
|
|
requiredText: z.string(),
|
|
children: z.array(z.object({ optionalChild: z.string().optional() })),
|
|
});
|
|
const nativeSchema = z.toJSONSchema(schema);
|
|
|
|
expect(normaliseOpenAITransportResponse({
|
|
optionalText: null,
|
|
nullableText: null,
|
|
requiredText: null,
|
|
children: [{ optionalChild: null }],
|
|
}, schema, nativeSchema)).toEqual({
|
|
nullableText: null,
|
|
requiredText: null,
|
|
children: [{}],
|
|
});
|
|
});
|
|
|
|
it("uses the Responses API with the canonical strict reconstruction schema", async () => {
|
|
const fetchSpy = vi.fn().mockResolvedValue({
|
|
ok: true,
|
|
json: async () => ({ output_text: '{"reconstruction":"result"}' }),
|
|
});
|
|
const provider = createOpenAIReconstructionProvider({
|
|
apiKey: "test-key",
|
|
fetchImpl: fetchSpy,
|
|
});
|
|
|
|
const result = await provider.generateReconstruction(
|
|
"current reconstruction prompt",
|
|
);
|
|
const request = JSON.parse(fetchSpy.mock.calls[0][1].body);
|
|
const schemaText = JSON.stringify(request.text.format.schema);
|
|
|
|
expect(fetchSpy).toHaveBeenCalledWith(
|
|
"https://api.openai.com/v1/responses",
|
|
expect.objectContaining({ method: "POST" }),
|
|
);
|
|
expect(request).toMatchObject({
|
|
model: "gpt-5.6-terra",
|
|
input: "current reconstruction prompt",
|
|
text: {
|
|
format: {
|
|
type: "json_schema",
|
|
name: "reconstruction",
|
|
strict: true,
|
|
},
|
|
},
|
|
});
|
|
expect(schemaText).toContain("relationship");
|
|
expect(schemaText).toContain("evidenceType");
|
|
expect(schemaText).toContain("primaryType");
|
|
expect(schemaText).toContain("secondaryTypes");
|
|
expect(schemaText).toContain("confidence");
|
|
expect(schemaText).toContain("importance");
|
|
expect(request.text.format.schema).not.toEqual(z.toJSONSchema(reconstructionV2Schema));
|
|
expect(result).toEqual({
|
|
response: { reconstruction: "result" },
|
|
providerApiPath: "/v1/responses",
|
|
});
|
|
});
|
|
|
|
it("surfaces Responses API failures without inventing reconstruction content", async () => {
|
|
const provider = createOpenAIReconstructionProvider({
|
|
apiKey: "test-key",
|
|
fetchImpl: vi.fn().mockResolvedValue({
|
|
ok: false,
|
|
status: 429,
|
|
text: async () => "rate limited",
|
|
}),
|
|
});
|
|
|
|
await expect(provider.generateReconstruction("prompt")).rejects.toMatchObject({
|
|
providerApiPath: "/v1/responses",
|
|
message: "OpenAI Responses API returned 429: rate limited",
|
|
});
|
|
});
|
|
}); |