import { describe, expect, it, vi } from "vitest"; import { __resetChatSupportForTests, createOpenAIStrictSchema, createOpenAIReconstructionProvider, getProvider, getProviderModelName, normaliseOpenAITransportResponse, reconstructionJsonSchema, } from "@/lib/llm/provider.js"; import { reconstructionV2Schema } from "@/lib/reconstruction/schema.js"; import { focusedDeconstructJsonSchema } from "@/lib/graph/focused-investigation.js"; import { z } from "zod"; describe("OllamaLlmProvider chat capability detection", () => { it("keeps Ollama and its configured model when no experiment provider is selected", () => { const previousExperimentProvider = process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER; const previousModel = process.env.OLLAMA_MODEL; const previousOpenAIKey = process.env.OPENAI_API_KEY; delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER; process.env.OLLAMA_MODEL = "qwen-default"; process.env.OPENAI_API_KEY = "present-but-not-selected"; try { expect(getProvider().constructor.name).toBe("OllamaLlmProvider"); expect(getProviderModelName()).toBe("qwen-default"); } finally { if (previousExperimentProvider === undefined) delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER; else process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER = previousExperimentProvider; if (previousModel === undefined) delete process.env.OLLAMA_MODEL; else process.env.OLLAMA_MODEL = previousModel; if (previousOpenAIKey === undefined) delete process.env.OPENAI_API_KEY; else process.env.OPENAI_API_KEY = previousOpenAIKey; } }); it("selects the existing OpenAI provider and Terra only for the explicit experiment configuration", () => { const previousExperimentProvider = process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER; const previousModel = process.env.OLLAMA_MODEL; const previousOpenAIKey = process.env.OPENAI_API_KEY; process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER = "openai"; process.env.OPENAI_API_KEY = "test-key"; process.env.OLLAMA_MODEL = "qwen-default"; try { expect(getProvider().constructor.name).toBe("OpenAIReconstructionProvider"); expect(getProviderModelName()).toBe("gpt-5.6-terra"); } finally { if (previousExperimentProvider === undefined) delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER; else process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER = previousExperimentProvider; if (previousModel === undefined) delete process.env.OLLAMA_MODEL; else process.env.OLLAMA_MODEL = previousModel; if (previousOpenAIKey === undefined) delete process.env.OPENAI_API_KEY; else process.env.OPENAI_API_KEY = previousOpenAIKey; } }); it("uses the configured model for the chat probe and keeps the chat path", async () => { const originalBaseUrl = process.env.OLLAMA_BASE_URL; const fetchSpy = vi.fn() .mockResolvedValueOnce({ ok: true, text: async () => "" }) .mockResolvedValueOnce({ ok: true, json: async () => ({ message: { content: "{}" } }), }); const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout"); vi.stubGlobal("fetch", fetchSpy); process.env.OLLAMA_BASE_URL = "http://ollama.test"; try { __resetChatSupportForTests(); const result = await getProvider().generateReconstruction("prompt", "configured-model"); expect(JSON.parse(fetchSpy.mock.calls[0][1].body)).toMatchObject({ model: "configured-model", stream: false, }); expect(fetchSpy.mock.calls[0][0]).toBe("http://ollama.test/api/chat"); expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/chat"); expect(fetchSpy.mock.calls[1][0]).not.toContain("/api/generate"); expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), 300000); const chatRequest = JSON.parse(fetchSpy.mock.calls[1][1].body); expect(chatRequest).toMatchObject({ model: "configured-model", messages: [{ role: "user", content: "prompt" }], stream: false, }); expect(chatRequest.format).toEqual(reconstructionJsonSchema); expect(result).toMatchObject({ response: {}, providerApiPath: "/api/chat", providerExecution: { chatCapabilityDetected: true, chatRequestAttempted: true, chatRequestSucceeded: true, generateRequestAttempted: false, }, }); } finally { setTimeoutSpy.mockRestore(); vi.unstubAllGlobals(); if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL; else process.env.OLLAMA_BASE_URL = originalBaseUrl; } }); it("uses a supplied structured-output schema for the chat request", async () => { const originalBaseUrl = process.env.OLLAMA_BASE_URL; const outputSchema = { type: "object", properties: { focused: { type: "string" } }, required: ["focused"], }; const fetchSpy = vi.fn() .mockResolvedValueOnce({ ok: true, text: async () => "" }) .mockResolvedValueOnce({ ok: true, json: async () => ({ message: { content: "{}" } }) }); vi.stubGlobal("fetch", fetchSpy); process.env.OLLAMA_BASE_URL = "http://ollama.test"; try { __resetChatSupportForTests(); await getProvider().generateReconstruction("prompt", "configured-model", outputSchema); expect(JSON.parse(fetchSpy.mock.calls[1][1].body).format).toEqual(outputSchema); } finally { vi.unstubAllGlobals(); if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL; else process.env.OLLAMA_BASE_URL = originalBaseUrl; } }); it("reports chat-skipped generate fallback execution", async () => { const originalBaseUrl = process.env.OLLAMA_BASE_URL; const fetchSpy = vi.fn() .mockResolvedValueOnce({ ok: false, status: 501, text: async () => "" }) .mockResolvedValueOnce({ ok: false, status: 500, text: async () => "failure" }); vi.stubGlobal("fetch", fetchSpy); process.env.OLLAMA_BASE_URL = "http://ollama.test"; try { __resetChatSupportForTests(); await expect( getProvider().generateReconstruction("prompt", "configured-model"), ).rejects.toMatchObject({ providerApiPath: "/api/generate", providerExecution: { chatCapabilityDetected: false, chatRequestAttempted: false, chatRequestSucceeded: false, generateRequestAttempted: true, }, }); expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/generate"); } finally { vi.unstubAllGlobals(); if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL; else process.env.OLLAMA_BASE_URL = originalBaseUrl; } }); it("reports chat-attempt-failed generate fallback execution", async () => { const originalBaseUrl = process.env.OLLAMA_BASE_URL; const fetchSpy = vi.fn() .mockResolvedValueOnce({ ok: true, text: async () => "" }) .mockResolvedValueOnce({ ok: false, text: async () => "" }) .mockResolvedValueOnce({ ok: false, status: 500, text: async () => "failure" }); vi.stubGlobal("fetch", fetchSpy); process.env.OLLAMA_BASE_URL = "http://ollama.test"; try { __resetChatSupportForTests(); await expect( getProvider().generateReconstruction("prompt", "configured-model"), ).rejects.toMatchObject({ providerApiPath: "/api/generate", providerExecution: { chatCapabilityDetected: true, chatRequestAttempted: true, chatRequestSucceeded: false, generateRequestAttempted: true, }, }); expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/chat"); expect(fetchSpy.mock.calls[2][0]).toBe("http://ollama.test/api/generate"); } finally { vi.unstubAllGlobals(); if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL; else process.env.OLLAMA_BASE_URL = originalBaseUrl; } }); }); describe("OpenAI reconstruction provider experiment seam", () => { function assertStrictObjectInvariant(schema, root = schema, visited = new Set()) { if (!schema || typeof schema !== "object" || visited.has(schema)) return; visited.add(schema); if (schema.type === "object" && !schema.properties) { expect(schema.additionalProperties).toBe(false); } if (schema.properties) { expect(schema.additionalProperties).toBe(false); expect(new Set(schema.required)).toEqual(new Set(Object.keys(schema.properties))); Object.values(schema.properties).forEach((property) => assertStrictObjectInvariant(property, root, visited), ); } if (schema.items) assertStrictObjectInvariant(schema.items, root, visited); for (const branch of [ ...(schema.anyOf ?? []), ...(schema.oneOf ?? []), ...(schema.allOf ?? []), ]) { assertStrictObjectInvariant(branch, root, visited); } for (const definition of Object.values(schema.$defs ?? schema.definitions ?? {})) { assertStrictObjectInvariant(definition, root, visited); } } it("projects canonical optional fields as required but nullable", () => { const nativeSchema = z.toJSONSchema(reconstructionV2Schema); const projectedSchema = createOpenAIStrictSchema( nativeSchema, reconstructionV2Schema, ); function resolveLocalRef(schema, root = projectedSchema) { if (!schema.$ref) return schema; return schema.$ref .replace(/^#\//, "") .split("/") .reduce((value, key) => value?.[key], root); } function acceptsNull(schema, root = projectedSchema) { const resolved = resolveLocalRef(schema, root); return ( resolved?.type === "null" || (Array.isArray(resolved?.type) && resolved.type.includes("null")) || [...(resolved?.anyOf ?? []), ...(resolved?.oneOf ?? [])].some((branch) => acceptsNull(branch, root), ) ); } function assertAllPropertiesRequired(schema, root = projectedSchema) { const resolved = resolveLocalRef(schema, root); if (resolved?.properties) { expect(resolved.required).toEqual(Object.keys(resolved.properties)); expect(resolved.additionalProperties).toBe(false); Object.values(resolved.properties).forEach((property) => assertAllPropertiesRequired(property, root)); } (resolved?.anyOf ?? []).forEach((branch) => assertAllPropertiesRequired(branch, root)); (resolved?.oneOf ?? []).forEach((branch) => assertAllPropertiesRequired(branch, root)); if (resolved?.items) assertAllPropertiesRequired(resolved.items, root); } assertAllPropertiesRequired(projectedSchema); const transition = projectedSchema.properties.reconstruction.properties.unexplainedTransitions.items; expect(transition.required).toContain("entity"); expect(acceptsNull(transition.properties.entity)).toBe(true); const inputClassification = resolveLocalRef( projectedSchema.properties.inputClassification, ); expect(inputClassification.required).toContain("secondaryTypes"); expect(acceptsNull(inputClassification.properties.secondaryTypes)).toBe(true); expect(acceptsNull(transition.properties.id)).toBe(false); }); it("removes only optional transport null placeholders", () => { const schema = z.object({ optionalText: z.string().optional(), nullableText: z.string().nullable(), requiredText: z.string(), children: z.array(z.object({ optionalChild: z.string().optional() })), }); const nativeSchema = z.toJSONSchema(schema); expect(normaliseOpenAITransportResponse({ optionalText: null, nullableText: null, requiredText: null, children: [{ optionalChild: null }], }, schema, nativeSchema)).toEqual({ nullableText: null, requiredText: null, children: [{}], }); }); it("uses the Responses API with the canonical strict reconstruction schema", async () => { const fetchSpy = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ output_text: '{"reconstruction":"result"}' }), }); const provider = createOpenAIReconstructionProvider({ apiKey: "test-key", fetchImpl: fetchSpy, }); const result = await provider.generateReconstruction( "current reconstruction prompt", ); const request = JSON.parse(fetchSpy.mock.calls[0][1].body); const schemaText = JSON.stringify(request.text.format.schema); expect(fetchSpy).toHaveBeenCalledWith( "https://api.openai.com/v1/responses", expect.objectContaining({ method: "POST" }), ); expect(request).toMatchObject({ model: "gpt-5.6-terra", input: "current reconstruction prompt", text: { format: { type: "json_schema", name: "reconstruction", strict: true, }, }, }); expect(schemaText).toContain("relationship"); expect(schemaText).toContain("evidenceType"); expect(schemaText).toContain("primaryType"); expect(schemaText).toContain("secondaryTypes"); expect(schemaText).toContain("confidence"); expect(schemaText).toContain("importance"); expect(request.text.format.schema).not.toEqual(z.toJSONSchema(reconstructionV2Schema)); expect(result).toEqual({ response: { reconstruction: "result" }, providerApiPath: "/v1/responses", }); }); it("projects a caller-supplied focused schema for OpenAI structured output", async () => { const fetchSpy = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ output_text: JSON.stringify({ targetNodeId: "target", observations: [], uncertainties: [], assumptions: [], relationships: [], possibleFollowUpQuestions: [], }), }), }); const provider = createOpenAIReconstructionProvider({ apiKey: "test-key", fetchImpl: fetchSpy }); const result = await provider.generateReconstruction( "focused prompt", "gpt-5.6-terra", focusedDeconstructJsonSchema, ); const request = JSON.parse(fetchSpy.mock.calls[0][1].body); const schema = request.text.format.schema; const serializedSchema = JSON.stringify(schema); const reparsedSchema = JSON.parse(serializedSchema); expect(schema.required).toEqual([ "targetNodeId", "observations", "uncertainties", "assumptions", "relationships", "possibleFollowUpQuestions", ]); expect(Object.keys(schema.properties)).toEqual(schema.required); expect(schema.additionalProperties).toBe(false); expect(schema.properties).toHaveProperty("targetNodeId"); expect(schema.properties.relationships.items.additionalProperties).toBe(false); expect(schema.properties).not.toHaveProperty("reconstruction"); expect(schema.properties).not.toHaveProperty("inputClassification"); expect(result.response).toEqual({ targetNodeId: "target", observations: [], uncertainties: [], assumptions: [], relationships: [], possibleFollowUpQuestions: [], }); expect(new Set(schema.required)).toEqual(new Set(Object.keys(schema.properties))); expect(schema.properties).toHaveProperty("relationships"); expect(schema.required).toContain("relationships"); assertStrictObjectInvariant(schema); expect(Object.prototype.hasOwnProperty.call(schema.properties, "relationships")).toBe(true); expect(reparsedSchema.required).toEqual(schema.required); expect(Object.keys(reparsedSchema.properties)).toEqual(Object.keys(schema.properties)); expect(Object.prototype.hasOwnProperty.call(reparsedSchema.properties, "relationships")).toBe(true); expect(reparsedSchema.properties.relationships).toEqual(schema.properties.relationships); expect(reparsedSchema.additionalProperties).toBe(false); assertStrictObjectInvariant(reparsedSchema); }); it("emits one safe final-schema trace only when explicitly enabled", async () => { const previousTraceFlag = process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA; const infoSpy = vi.spyOn(console, "info").mockImplementation(() => {}); const fetchSpy = vi.fn().mockResolvedValue({ ok: true, json: async () => ({ output_text: JSON.stringify({ targetNodeId: "target", observations: [], uncertainties: [], assumptions: [], relationships: [], possibleFollowUpQuestions: [], }) }), }); process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA = "1"; try { const provider = createOpenAIReconstructionProvider({ apiKey: "test-key", fetchImpl: fetchSpy }); await provider.generateReconstruction("focused prompt", "gpt-5.6-terra", focusedDeconstructJsonSchema); expect(infoSpy).toHaveBeenCalledTimes(1); const [label, trace] = infoSpy.mock.calls[0]; expect(label).toBe("[confidence-engine][openai-schema-trace]"); expect(trace).toMatchObject({ model: "gpt-5.6-terra", formatType: "json_schema", formatName: "reconstruction", strict: true, rootSetsEqual: true, relationshipsInProperties: true, relationshipsInRequired: true, relationshipsItemAdditionalProperties: false, rootAdditionalProperties: false, recursiveStrictInvariant: true, }); expect(JSON.stringify(trace)).not.toContain("focused prompt"); } finally { infoSpy.mockRestore(); if (previousTraceFlag === undefined) delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA; else process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA = previousTraceFlag; } }); it("does not emit a schema trace when the flag is absent", async () => { const previousTraceFlag = process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA; const infoSpy = vi.spyOn(console, "info").mockImplementation(() => {}); delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA; const provider = createOpenAIReconstructionProvider({ apiKey: "test-key", fetchImpl: vi.fn().mockResolvedValue({ ok: true, json: async () => ({ output_text: "{}" }) }), }); try { await provider.generateReconstruction("prompt"); expect(infoSpy).not.toHaveBeenCalled(); } finally { infoSpy.mockRestore(); if (previousTraceFlag === undefined) delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA; else process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA = previousTraceFlag; } }); it("recursively projects arbitrary nested object schemas as strict", () => { const alternateSchema = { type: "object", properties: { groups: { type: "array", items: { type: "object", properties: { details: { type: "object", properties: { value: { type: "string" } }, }, }, }, }, }, }; const projected = createOpenAIStrictSchema(alternateSchema, null); expect(projected.additionalProperties).toBe(false); expect(projected.properties.groups.items.additionalProperties).toBe(false); expect(projected.properties.groups.items.properties.details.additionalProperties).toBe(false); }); it("projects property-less objects nested in arrays as strict", () => { const projected = createOpenAIStrictSchema({ type: "object", properties: { relationships: { type: "array", items: { type: "object" } }, }, }, null); expect(projected.properties.relationships.items).toEqual({ type: "object", additionalProperties: false, }); }); it("extracts ordered output_text parts from raw Responses output", async () => { const provider = createOpenAIReconstructionProvider({ apiKey: "test-key", fetchImpl: vi.fn().mockResolvedValue({ ok: true, json: async () => ({ output: [ { type: "reasoning", content: [] }, { type: "message", role: "assistant", content: [ { type: "output_text", text: '{"reconstruction":' }, { type: "output_text", text: '"result"}' }, ], }, ], }), }), }); await expect(provider.generateReconstruction("prompt")).resolves.toEqual({ response: { reconstruction: "result" }, providerApiPath: "/v1/responses", }); }); it("preserves output_text convenience responses", async () => { const provider = createOpenAIReconstructionProvider({ apiKey: "test-key", fetchImpl: vi.fn().mockResolvedValue({ ok: true, json: async () => ({ output_text: '{"reconstruction":"result"}' }), }), }); await expect(provider.generateReconstruction("prompt")).resolves.toEqual({ response: { reconstruction: "result" }, providerApiPath: "/v1/responses", }); }); it("fails safely when a raw Responses result has no output text", async () => { const provider = createOpenAIReconstructionProvider({ apiKey: "test-key", fetchImpl: vi.fn().mockResolvedValue({ ok: true, json: async () => ({ output: [{ type: "message", content: [{ type: "refusal" }] }], }), }), }); await expect(provider.generateReconstruction("prompt")).rejects.toMatchObject({ providerApiPath: "/v1/responses", message: expect.stringContaining("refusal: true"), }); }); it("surfaces Responses API failures without inventing reconstruction content", async () => { const provider = createOpenAIReconstructionProvider({ apiKey: "test-key", fetchImpl: vi.fn().mockResolvedValue({ ok: false, status: 429, text: async () => "rate limited", }), }); await expect(provider.generateReconstruction("prompt")).rejects.toMatchObject({ providerApiPath: "/v1/responses", message: "OpenAI Responses API returned 429: rate limited", }); }); });