470 lines
17 KiB
JavaScript
470 lines
17 KiB
JavaScript
import { describe, expect, it, vi } from "vitest";
|
|
import {
|
|
__resetChatSupportForTests,
|
|
createOpenAIStrictSchema,
|
|
createOpenAIReconstructionProvider,
|
|
getProvider,
|
|
getProviderModelName,
|
|
normaliseOpenAITransportResponse,
|
|
reconstructionJsonSchema,
|
|
} from "@/lib/llm/provider.js";
|
|
import { reconstructionV2Schema } from "@/lib/reconstruction/schema.js";
|
|
import { focusedDeconstructJsonSchema } from "@/lib/graph/focused-investigation.js";
|
|
import { z } from "zod";
|
|
|
|
describe("OllamaLlmProvider chat capability detection", () => {
|
|
it("keeps Ollama and its configured model when no experiment provider is selected", () => {
|
|
const previousExperimentProvider = process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER;
|
|
const previousModel = process.env.OLLAMA_MODEL;
|
|
const previousOpenAIKey = process.env.OPENAI_API_KEY;
|
|
delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER;
|
|
process.env.OLLAMA_MODEL = "qwen-default";
|
|
process.env.OPENAI_API_KEY = "present-but-not-selected";
|
|
|
|
try {
|
|
expect(getProvider().constructor.name).toBe("OllamaLlmProvider");
|
|
expect(getProviderModelName()).toBe("qwen-default");
|
|
} finally {
|
|
if (previousExperimentProvider === undefined) delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER;
|
|
else process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER = previousExperimentProvider;
|
|
if (previousModel === undefined) delete process.env.OLLAMA_MODEL;
|
|
else process.env.OLLAMA_MODEL = previousModel;
|
|
if (previousOpenAIKey === undefined) delete process.env.OPENAI_API_KEY;
|
|
else process.env.OPENAI_API_KEY = previousOpenAIKey;
|
|
}
|
|
});
|
|
|
|
it("selects the existing OpenAI provider and Terra only for the explicit experiment configuration", () => {
|
|
const previousExperimentProvider = process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER;
|
|
const previousModel = process.env.OLLAMA_MODEL;
|
|
const previousOpenAIKey = process.env.OPENAI_API_KEY;
|
|
process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER = "openai";
|
|
process.env.OPENAI_API_KEY = "test-key";
|
|
process.env.OLLAMA_MODEL = "qwen-default";
|
|
|
|
try {
|
|
expect(getProvider().constructor.name).toBe("OpenAIReconstructionProvider");
|
|
expect(getProviderModelName()).toBe("gpt-5.6-terra");
|
|
} finally {
|
|
if (previousExperimentProvider === undefined) delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER;
|
|
else process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER = previousExperimentProvider;
|
|
if (previousModel === undefined) delete process.env.OLLAMA_MODEL;
|
|
else process.env.OLLAMA_MODEL = previousModel;
|
|
if (previousOpenAIKey === undefined) delete process.env.OPENAI_API_KEY;
|
|
else process.env.OPENAI_API_KEY = previousOpenAIKey;
|
|
}
|
|
});
|
|
|
|
it("uses the configured model for the chat probe and keeps the chat path", async () => {
|
|
const originalBaseUrl = process.env.OLLAMA_BASE_URL;
|
|
const fetchSpy = vi.fn()
|
|
.mockResolvedValueOnce({ ok: true, text: async () => "" })
|
|
.mockResolvedValueOnce({
|
|
ok: true,
|
|
json: async () => ({ message: { content: "{}" } }),
|
|
});
|
|
const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout");
|
|
vi.stubGlobal("fetch", fetchSpy);
|
|
process.env.OLLAMA_BASE_URL = "http://ollama.test";
|
|
|
|
try {
|
|
__resetChatSupportForTests();
|
|
const result = await getProvider().generateReconstruction("prompt", "configured-model");
|
|
|
|
expect(JSON.parse(fetchSpy.mock.calls[0][1].body)).toMatchObject({
|
|
model: "configured-model",
|
|
stream: false,
|
|
});
|
|
expect(fetchSpy.mock.calls[0][0]).toBe("http://ollama.test/api/chat");
|
|
expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/chat");
|
|
expect(fetchSpy.mock.calls[1][0]).not.toContain("/api/generate");
|
|
expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), 300000);
|
|
const chatRequest = JSON.parse(fetchSpy.mock.calls[1][1].body);
|
|
expect(chatRequest).toMatchObject({
|
|
model: "configured-model",
|
|
messages: [{ role: "user", content: "prompt" }],
|
|
stream: false,
|
|
});
|
|
expect(chatRequest.format).toEqual(reconstructionJsonSchema);
|
|
expect(result).toMatchObject({
|
|
response: {},
|
|
providerApiPath: "/api/chat",
|
|
providerExecution: {
|
|
chatCapabilityDetected: true,
|
|
chatRequestAttempted: true,
|
|
chatRequestSucceeded: true,
|
|
generateRequestAttempted: false,
|
|
},
|
|
});
|
|
} finally {
|
|
setTimeoutSpy.mockRestore();
|
|
vi.unstubAllGlobals();
|
|
if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
|
|
else process.env.OLLAMA_BASE_URL = originalBaseUrl;
|
|
}
|
|
});
|
|
|
|
it("uses a supplied structured-output schema for the chat request", async () => {
|
|
const originalBaseUrl = process.env.OLLAMA_BASE_URL;
|
|
const outputSchema = {
|
|
type: "object",
|
|
properties: { focused: { type: "string" } },
|
|
required: ["focused"],
|
|
};
|
|
const fetchSpy = vi.fn()
|
|
.mockResolvedValueOnce({ ok: true, text: async () => "" })
|
|
.mockResolvedValueOnce({ ok: true, json: async () => ({ message: { content: "{}" } }) });
|
|
vi.stubGlobal("fetch", fetchSpy);
|
|
process.env.OLLAMA_BASE_URL = "http://ollama.test";
|
|
|
|
try {
|
|
__resetChatSupportForTests();
|
|
await getProvider().generateReconstruction("prompt", "configured-model", outputSchema);
|
|
expect(JSON.parse(fetchSpy.mock.calls[1][1].body).format).toEqual(outputSchema);
|
|
} finally {
|
|
vi.unstubAllGlobals();
|
|
if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
|
|
else process.env.OLLAMA_BASE_URL = originalBaseUrl;
|
|
}
|
|
});
|
|
|
|
it("reports chat-skipped generate fallback execution", async () => {
|
|
const originalBaseUrl = process.env.OLLAMA_BASE_URL;
|
|
const fetchSpy = vi.fn()
|
|
.mockResolvedValueOnce({ ok: false, status: 501, text: async () => "" })
|
|
.mockResolvedValueOnce({ ok: false, status: 500, text: async () => "failure" });
|
|
vi.stubGlobal("fetch", fetchSpy);
|
|
process.env.OLLAMA_BASE_URL = "http://ollama.test";
|
|
|
|
try {
|
|
__resetChatSupportForTests();
|
|
await expect(
|
|
getProvider().generateReconstruction("prompt", "configured-model"),
|
|
).rejects.toMatchObject({
|
|
providerApiPath: "/api/generate",
|
|
providerExecution: {
|
|
chatCapabilityDetected: false,
|
|
chatRequestAttempted: false,
|
|
chatRequestSucceeded: false,
|
|
generateRequestAttempted: true,
|
|
},
|
|
});
|
|
expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/generate");
|
|
} finally {
|
|
vi.unstubAllGlobals();
|
|
if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
|
|
else process.env.OLLAMA_BASE_URL = originalBaseUrl;
|
|
}
|
|
});
|
|
|
|
it("reports chat-attempt-failed generate fallback execution", async () => {
|
|
const originalBaseUrl = process.env.OLLAMA_BASE_URL;
|
|
const fetchSpy = vi.fn()
|
|
.mockResolvedValueOnce({ ok: true, text: async () => "" })
|
|
.mockResolvedValueOnce({ ok: false, text: async () => "" })
|
|
.mockResolvedValueOnce({ ok: false, status: 500, text: async () => "failure" });
|
|
vi.stubGlobal("fetch", fetchSpy);
|
|
process.env.OLLAMA_BASE_URL = "http://ollama.test";
|
|
|
|
try {
|
|
__resetChatSupportForTests();
|
|
await expect(
|
|
getProvider().generateReconstruction("prompt", "configured-model"),
|
|
).rejects.toMatchObject({
|
|
providerApiPath: "/api/generate",
|
|
providerExecution: {
|
|
chatCapabilityDetected: true,
|
|
chatRequestAttempted: true,
|
|
chatRequestSucceeded: false,
|
|
generateRequestAttempted: true,
|
|
},
|
|
});
|
|
expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/chat");
|
|
expect(fetchSpy.mock.calls[2][0]).toBe("http://ollama.test/api/generate");
|
|
} finally {
|
|
vi.unstubAllGlobals();
|
|
if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
|
|
else process.env.OLLAMA_BASE_URL = originalBaseUrl;
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("OpenAI reconstruction provider experiment seam", () => {
|
|
it("projects canonical optional fields as required but nullable", () => {
|
|
const nativeSchema = z.toJSONSchema(reconstructionV2Schema);
|
|
const projectedSchema = createOpenAIStrictSchema(
|
|
nativeSchema,
|
|
reconstructionV2Schema,
|
|
);
|
|
|
|
function resolveLocalRef(schema, root = projectedSchema) {
|
|
if (!schema.$ref) return schema;
|
|
return schema.$ref
|
|
.replace(/^#\//, "")
|
|
.split("/")
|
|
.reduce((value, key) => value?.[key], root);
|
|
}
|
|
|
|
function acceptsNull(schema, root = projectedSchema) {
|
|
const resolved = resolveLocalRef(schema, root);
|
|
return (
|
|
resolved?.type === "null" ||
|
|
(Array.isArray(resolved?.type) && resolved.type.includes("null")) ||
|
|
[...(resolved?.anyOf ?? []), ...(resolved?.oneOf ?? [])].some((branch) =>
|
|
acceptsNull(branch, root),
|
|
)
|
|
);
|
|
}
|
|
|
|
function assertAllPropertiesRequired(schema, root = projectedSchema) {
|
|
const resolved = resolveLocalRef(schema, root);
|
|
if (resolved?.properties) {
|
|
expect(resolved.required).toEqual(Object.keys(resolved.properties));
|
|
expect(resolved.additionalProperties).toBe(false);
|
|
Object.values(resolved.properties).forEach((property) => assertAllPropertiesRequired(property, root));
|
|
}
|
|
(resolved?.anyOf ?? []).forEach((branch) => assertAllPropertiesRequired(branch, root));
|
|
(resolved?.oneOf ?? []).forEach((branch) => assertAllPropertiesRequired(branch, root));
|
|
if (resolved?.items) assertAllPropertiesRequired(resolved.items, root);
|
|
}
|
|
|
|
assertAllPropertiesRequired(projectedSchema);
|
|
const transition = projectedSchema.properties.reconstruction.properties.unexplainedTransitions.items;
|
|
expect(transition.required).toContain("entity");
|
|
expect(acceptsNull(transition.properties.entity)).toBe(true);
|
|
const inputClassification = resolveLocalRef(
|
|
projectedSchema.properties.inputClassification,
|
|
);
|
|
expect(inputClassification.required).toContain("secondaryTypes");
|
|
expect(acceptsNull(inputClassification.properties.secondaryTypes)).toBe(true);
|
|
expect(acceptsNull(transition.properties.id)).toBe(false);
|
|
});
|
|
|
|
it("removes only optional transport null placeholders", () => {
|
|
const schema = z.object({
|
|
optionalText: z.string().optional(),
|
|
nullableText: z.string().nullable(),
|
|
requiredText: z.string(),
|
|
children: z.array(z.object({ optionalChild: z.string().optional() })),
|
|
});
|
|
const nativeSchema = z.toJSONSchema(schema);
|
|
|
|
expect(normaliseOpenAITransportResponse({
|
|
optionalText: null,
|
|
nullableText: null,
|
|
requiredText: null,
|
|
children: [{ optionalChild: null }],
|
|
}, schema, nativeSchema)).toEqual({
|
|
nullableText: null,
|
|
requiredText: null,
|
|
children: [{}],
|
|
});
|
|
});
|
|
|
|
it("uses the Responses API with the canonical strict reconstruction schema", async () => {
|
|
const fetchSpy = vi.fn().mockResolvedValue({
|
|
ok: true,
|
|
json: async () => ({ output_text: '{"reconstruction":"result"}' }),
|
|
});
|
|
const provider = createOpenAIReconstructionProvider({
|
|
apiKey: "test-key",
|
|
fetchImpl: fetchSpy,
|
|
});
|
|
|
|
const result = await provider.generateReconstruction(
|
|
"current reconstruction prompt",
|
|
);
|
|
const request = JSON.parse(fetchSpy.mock.calls[0][1].body);
|
|
const schemaText = JSON.stringify(request.text.format.schema);
|
|
|
|
expect(fetchSpy).toHaveBeenCalledWith(
|
|
"https://api.openai.com/v1/responses",
|
|
expect.objectContaining({ method: "POST" }),
|
|
);
|
|
expect(request).toMatchObject({
|
|
model: "gpt-5.6-terra",
|
|
input: "current reconstruction prompt",
|
|
text: {
|
|
format: {
|
|
type: "json_schema",
|
|
name: "reconstruction",
|
|
strict: true,
|
|
},
|
|
},
|
|
});
|
|
expect(schemaText).toContain("relationship");
|
|
expect(schemaText).toContain("evidenceType");
|
|
expect(schemaText).toContain("primaryType");
|
|
expect(schemaText).toContain("secondaryTypes");
|
|
expect(schemaText).toContain("confidence");
|
|
expect(schemaText).toContain("importance");
|
|
expect(request.text.format.schema).not.toEqual(z.toJSONSchema(reconstructionV2Schema));
|
|
expect(result).toEqual({
|
|
response: { reconstruction: "result" },
|
|
providerApiPath: "/v1/responses",
|
|
});
|
|
});
|
|
|
|
it("projects a caller-supplied focused schema for OpenAI structured output", async () => {
|
|
const fetchSpy = vi.fn().mockResolvedValue({
|
|
ok: true,
|
|
json: async () => ({
|
|
output_text: JSON.stringify({
|
|
targetNodeId: "target",
|
|
observations: [],
|
|
uncertainties: [],
|
|
assumptions: [],
|
|
relationships: [],
|
|
possibleFollowUpQuestions: [],
|
|
}),
|
|
}),
|
|
});
|
|
const provider = createOpenAIReconstructionProvider({ apiKey: "test-key", fetchImpl: fetchSpy });
|
|
|
|
const result = await provider.generateReconstruction(
|
|
"focused prompt",
|
|
"gpt-5.6-terra",
|
|
focusedDeconstructJsonSchema,
|
|
);
|
|
const request = JSON.parse(fetchSpy.mock.calls[0][1].body);
|
|
const schema = request.text.format.schema;
|
|
|
|
expect(schema.required).toEqual([
|
|
"targetNodeId",
|
|
"observations",
|
|
"uncertainties",
|
|
"assumptions",
|
|
"relationships",
|
|
"possibleFollowUpQuestions",
|
|
]);
|
|
expect(Object.keys(schema.properties)).toEqual(schema.required);
|
|
expect(schema.additionalProperties).toBe(false);
|
|
expect(schema.properties).toHaveProperty("targetNodeId");
|
|
expect(schema.properties.relationships.items.additionalProperties).toBe(false);
|
|
expect(schema.properties).not.toHaveProperty("reconstruction");
|
|
expect(schema.properties).not.toHaveProperty("inputClassification");
|
|
expect(result.response).toEqual({
|
|
targetNodeId: "target",
|
|
observations: [],
|
|
uncertainties: [],
|
|
assumptions: [],
|
|
relationships: [],
|
|
possibleFollowUpQuestions: [],
|
|
});
|
|
});
|
|
|
|
it("recursively projects arbitrary nested object schemas as strict", () => {
|
|
const alternateSchema = {
|
|
type: "object",
|
|
properties: {
|
|
groups: {
|
|
type: "array",
|
|
items: {
|
|
type: "object",
|
|
properties: {
|
|
details: {
|
|
type: "object",
|
|
properties: { value: { type: "string" } },
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
const projected = createOpenAIStrictSchema(alternateSchema, null);
|
|
|
|
expect(projected.additionalProperties).toBe(false);
|
|
expect(projected.properties.groups.items.additionalProperties).toBe(false);
|
|
expect(projected.properties.groups.items.properties.details.additionalProperties).toBe(false);
|
|
});
|
|
|
|
it("projects property-less objects nested in arrays as strict", () => {
|
|
const projected = createOpenAIStrictSchema({
|
|
type: "object",
|
|
properties: {
|
|
relationships: { type: "array", items: { type: "object" } },
|
|
},
|
|
}, null);
|
|
|
|
expect(projected.properties.relationships.items).toEqual({
|
|
type: "object",
|
|
additionalProperties: false,
|
|
});
|
|
});
|
|
|
|
it("extracts ordered output_text parts from raw Responses output", async () => {
|
|
const provider = createOpenAIReconstructionProvider({
|
|
apiKey: "test-key",
|
|
fetchImpl: vi.fn().mockResolvedValue({
|
|
ok: true,
|
|
json: async () => ({
|
|
output: [
|
|
{ type: "reasoning", content: [] },
|
|
{
|
|
type: "message",
|
|
role: "assistant",
|
|
content: [
|
|
{ type: "output_text", text: '{"reconstruction":' },
|
|
{ type: "output_text", text: '"result"}' },
|
|
],
|
|
},
|
|
],
|
|
}),
|
|
}),
|
|
});
|
|
|
|
await expect(provider.generateReconstruction("prompt")).resolves.toEqual({
|
|
response: { reconstruction: "result" },
|
|
providerApiPath: "/v1/responses",
|
|
});
|
|
});
|
|
|
|
it("preserves output_text convenience responses", async () => {
|
|
const provider = createOpenAIReconstructionProvider({
|
|
apiKey: "test-key",
|
|
fetchImpl: vi.fn().mockResolvedValue({
|
|
ok: true,
|
|
json: async () => ({ output_text: '{"reconstruction":"result"}' }),
|
|
}),
|
|
});
|
|
|
|
await expect(provider.generateReconstruction("prompt")).resolves.toEqual({
|
|
response: { reconstruction: "result" },
|
|
providerApiPath: "/v1/responses",
|
|
});
|
|
});
|
|
|
|
it("fails safely when a raw Responses result has no output text", async () => {
|
|
const provider = createOpenAIReconstructionProvider({
|
|
apiKey: "test-key",
|
|
fetchImpl: vi.fn().mockResolvedValue({
|
|
ok: true,
|
|
json: async () => ({
|
|
output: [{ type: "message", content: [{ type: "refusal" }] }],
|
|
}),
|
|
}),
|
|
});
|
|
|
|
await expect(provider.generateReconstruction("prompt")).rejects.toMatchObject({
|
|
providerApiPath: "/v1/responses",
|
|
message: expect.stringContaining("refusal: true"),
|
|
});
|
|
});
|
|
|
|
it("surfaces Responses API failures without inventing reconstruction content", async () => {
|
|
const provider = createOpenAIReconstructionProvider({
|
|
apiKey: "test-key",
|
|
fetchImpl: vi.fn().mockResolvedValue({
|
|
ok: false,
|
|
status: 429,
|
|
text: async () => "rate limited",
|
|
}),
|
|
});
|
|
|
|
await expect(provider.generateReconstruction("prompt")).rejects.toMatchObject({
|
|
providerApiPath: "/v1/responses",
|
|
message: "OpenAI Responses API returned 429: rate limited",
|
|
});
|
|
});
|
|
}); |