From 8dcdffce44442244e9cbc382f9bcdc7affdc23ea Mon Sep 17 00:00:00 2001 From: robbond Date: Tue, 13 Oct 2026 11:59:47 +0100 Subject: [PATCH] feat(experiments): support freeform provider responses --- lib/llm/provider.js | 75 ++++++++++++++++----- tests/llm/provider.test.js | 133 +++++++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 18 deletions(-) diff --git a/lib/llm/provider.js b/lib/llm/provider.js index e3caaec..3a3ceba 100644 --- a/lib/llm/provider.js +++ b/lib/llm/provider.js @@ -352,6 +352,9 @@ class OllamaLlmProvider { const baseUrl = process.env.OLLAMA_BASE_URL; if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set"); + // Explicit null signals "freeform text" — no JSON-schema constraint. + const useFreeformText = outputSchema === null; + let apiUsed = null; let chatSupported = false; let rawResponse = null; @@ -382,15 +385,19 @@ class OllamaLlmProvider { try { apiUsed = "/api/chat"; providerExecution.chatRequestAttempted = true; + const body = { + model: modelName, + messages: [{ role: "user", content: prompt }], + stream: false, + }; + // Explicit null → omit format constraint entirely. + if (outputSchema !== null) { + body.format = outputSchema ?? reconstructionJsonSchema; + } const res = await fetch(`${baseUrl}/api/chat`, { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - model: modelName, - messages: [{ role: "user", content: prompt }], - stream: false, - format: outputSchema ?? reconstructionJsonSchema, - }), + body: JSON.stringify(body), signal: requestAbort.signal, }); @@ -445,6 +452,11 @@ class OllamaLlmProvider { ? fullResponseData.response : JSON.stringify(fullResponseData); + // generate has no format constraint — treat freeform input as freeform output. + if (useFreeformText) { + apiUsed = "/api/generate"; + } + } catch (e) { if (apiUsed === "/api/generate") { const error = new Error( @@ -476,11 +488,11 @@ class OllamaLlmProvider { // ================================================================ if (rawResponse === "") { let diagInfo = ""; - + if (fullResponseData) { const keys = Object.keys(fullResponseData); diagInfo = "Keys in response: " + keys.join(", ") + "\n"; - + for (const key of keys) { const val = fullResponseData[key]; if (typeof val === "string") { @@ -501,6 +513,7 @@ class OllamaLlmProvider { "\nPossible causes:\n" + "- Check 'ollama list' — make sure the model name matches exactly what's installed\n" + "- The model may be corrupted. Try: ollama pull " + modelName + "\n" + + (useFreeformText ? "- No format constraint was sent — the model may not support freeform output.\n" : "") + "- This Ollama version does not support format:json — using prompt instructions only (reliability varies)\n" + "- If your model is very small (e.g., tinyllama, phi), try a larger one like llama3.1 or mistral" ); @@ -512,6 +525,14 @@ class OllamaLlmProvider { // ================================================================ // Step 5: Parse and return // ================================================================ + if (useFreeformText) { + return { + response: rawResponse, + providerApiPath: apiUsed, + providerExecution, + }; + } + try { return { response: recoverJson(rawResponse), @@ -553,17 +574,27 @@ class OpenAIReconstructionProvider { { signal, outputZodSchema } = {}, ) { if (!this.apiKey) throw new Error("OPENAI_API_KEY is not set"); - const structuredOutputSchema = outputSchema - ? createOpenAIStrictSchema(outputSchema, outputZodSchema ?? null) - : openAIReconstructionJsonSchema; - const format = { - type: "json_schema", - name: "reconstruction", - strict: true, - schema: structuredOutputSchema, - }; - traceOpenAISchema({ model: modelName, format, schema: structuredOutputSchema }); + // Explicit null → freeform text output (no JSON-schema constraint). + const useFreeformText = outputSchema === null; + + let format; + if (useFreeformText) { + format = { type: "text" }; + } else { + const structuredOutputSchema = outputSchema + ? createOpenAIStrictSchema(outputSchema, outputZodSchema ?? null) + : openAIReconstructionJsonSchema; + format = { + type: "json_schema", + name: "reconstruction", + strict: true, + schema: structuredOutputSchema, + }; + + traceOpenAISchema({ model: modelName, format, schema: structuredOutputSchema }); + } + const response = await this.fetchImpl("https://api.openai.com/v1/responses", { method: "POST", headers: { @@ -599,6 +630,14 @@ class OpenAIReconstructionProvider { throw error; } + // Freeform mode — return raw text without JSON recovery or transformation. + if (useFreeformText) { + return { + response: outputText, + providerApiPath: "/v1/responses", + }; + } + const recoveredResponse = recoverJson(outputText); const normalisedResponse = outputSchema && !outputZodSchema ? recoveredResponse diff --git a/tests/llm/provider.test.js b/tests/llm/provider.test.js index d5d598b..bceda7c 100644 --- a/tests/llm/provider.test.js +++ b/tests/llm/provider.test.js @@ -225,6 +225,139 @@ describe("OllamaLlmProvider chat capability detection", () => { }); }); +describe("Ollama freeform semantics", () => { + it("omits format constraint when outputSchema is explicitly null (chat path)", async () => { + const originalBaseUrl = process.env.OLLAMA_BASE_URL; + const fetchSpy = vi.fn() + .mockResolvedValueOnce({ ok: true, text: async () => "" }) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ message: { content: '{"freeform":"output"}' } }), + }); + vi.stubGlobal("fetch", fetchSpy); + process.env.OLLAMA_BASE_URL = "http://ollama.test"; + + try { + __resetChatSupportForTests(); + const provider = getProvider(); + const result = await provider.generateReconstruction("prompt", "model", null, { signal: undefined }); + + expect(result.response).toBe("{\"freeform\":\"output\"}"); + expect(result.providerApiPath).toBe("/api/chat"); + + // No format key in request body when schema is explicitly null + const requestBody = JSON.parse(fetchSpy.mock.calls[1][1].body); + expect(requestBody).not.toHaveProperty("format"); + } finally { + vi.unstubAllGlobals(); + if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL; + else process.env.OLLAMA_BASE_URL = originalBaseUrl; + } + }); + + it("omits format constraint when outputSchema is explicitly null (generate fallback)", async () => { + const originalBaseUrl = process.env.OLLAMA_BASE_URL; + const fetchSpy = vi.fn() + .mockResolvedValueOnce({ ok: false, status: 501, text: async () => "" }) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ response: "freeform text from /api/generate" }), + }); + vi.stubGlobal("fetch", fetchSpy); + process.env.OLLAMA_BASE_URL = "http://ollama.test"; + + try { + __resetChatSupportForTests(); + const provider = getProvider(); + const result = await provider.generateReconstruction("prompt", "model", null, { signal: undefined }); + + expect(result.response).toBe("freeform text from /api/generate"); + expect(result.providerApiPath).toBe("/api/generate"); + + // No format key in request body when schema is explicitly null + const requestBody = JSON.parse(fetchSpy.mock.calls[1][1].body); + expect(requestBody).not.toHaveProperty("format"); + } finally { + vi.unstubAllGlobals(); + if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL; + else process.env.OLLAMA_BASE_URL = originalBaseUrl; + } + }); + + it("still applies default schema when outputSchema is undefined", async () => { + const originalBaseUrl = process.env.OLLAMA_BASE_URL; + const fetchSpy = vi.fn() + .mockResolvedValueOnce({ ok: true, text: async () => "" }) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ message: { content: "{}" } }), + }); + vi.stubGlobal("fetch", fetchSpy); + process.env.OLLAMA_BASE_URL = "http://ollama.test"; + + try { + __resetChatSupportForTests(); + await getProvider().generateReconstruction("prompt", "model", undefined, { signal: undefined }); + + const requestBody = JSON.parse(fetchSpy.mock.calls[1][1].body); + // undefined → fallback to reconstructionJsonSchema (format IS present) + expect(requestBody.format).toBeDefined(); + } finally { + vi.unstubAllGlobals(); + if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL; + else process.env.OLLAMA_BASE_URL = originalBaseUrl; + } + }); +}); + +describe("OpenAI freeform semantics", () => { + it("uses text format when outputSchema is explicitly null", async () => { + const provider = createOpenAIReconstructionProvider({ + apiKey: "test-key", + fetchImpl: vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ output_text: "freeform text response" }), + }), + }); + + await provider.generateReconstruction("prompt", "model", null); + + const request = JSON.parse( + provider.fetchImpl.mock.calls[0][1].body, + ); + expect(request.text.format.type).toBe("text"); + }); + + it("returns raw text without JSON recovery when outputSchema is explicitly null", async () => { + const provider = createOpenAIReconstructionProvider({ + apiKey: "test-key", + fetchImpl: vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ output_text: "not-json-at-all {broken" }), + }), + }); + + const result = await provider.generateReconstruction("prompt", "model", null); + expect(result.response).toBe("not-json-at-all {broken"); + expect(result.providerApiPath).toBe("/v1/responses"); + }); + + it("still applies json_schema format when outputSchema is undefined", async () => { + const provider = createOpenAIReconstructionProvider({ + apiKey: "test-key", + fetchImpl: vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ output_text: "{}" }), + }), + }); + + await provider.generateReconstruction("prompt"); + + const request = JSON.parse(provider.fetchImpl.mock.calls[0][1].body); + expect(request.text.format.type).toBe("json_schema"); + }); +}); + describe("OpenAI reconstruction provider experiment seam", () => { function assertStrictObjectInvariant(schema, root = schema, visited = new Set()) { if (!schema || typeof schema !== "object" || visited.has(schema)) return;