robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(experiments): support freeform provider responses

Browse Source
This commit is contained in:
robbond
2026-10-13 11:59:47 +01:00
parent 74936851ce
commit 8dcdffce44
2 changed files with 190 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/llm/provider.js
+57 -18
View File
@@ -352,6 +352,9 @@ class OllamaLlmProvider {
const baseUrl = process.env.OLLAMA_BASE_URL;
if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set");
// Explicit null signals "freeform text" — no JSON-schema constraint.
const useFreeformText = outputSchema === null;
let apiUsed = null;
let chatSupported = false;
let rawResponse = null;
@@ -382,15 +385,19 @@ class OllamaLlmProvider {
try {
apiUsed = "/api/chat";
providerExecution.chatRequestAttempted = true;
const body = {
model: modelName,
messages: [{ role: "user", content: prompt }],
stream: false,
};
// Explicit null → omit format constraint entirely.
if (outputSchema !== null) {
body.format = outputSchema ?? reconstructionJsonSchema;
}
const res = await fetch(`${baseUrl}/api/chat`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
model: modelName,
messages: [{ role: "user", content: prompt }],
stream: false,
format: outputSchema ?? reconstructionJsonSchema,
}),
body: JSON.stringify(body),
signal: requestAbort.signal,
});
@@ -445,6 +452,11 @@ class OllamaLlmProvider {
? fullResponseData.response
: JSON.stringify(fullResponseData);
// generate has no format constraint — treat freeform input as freeform output.
if (useFreeformText) {
apiUsed = "/api/generate";
}
} catch (e) {
if (apiUsed === "/api/generate") {
const error = new Error(
@@ -476,11 +488,11 @@ class OllamaLlmProvider {
// ================================================================
if (rawResponse === "") {
let diagInfo = "";
if (fullResponseData) {
const keys = Object.keys(fullResponseData);
diagInfo = "Keys in response: " + keys.join(", ") + "\n";
for (const key of keys) {
const val = fullResponseData[key];
if (typeof val === "string") {
@@ -501,6 +513,7 @@ class OllamaLlmProvider {
"\nPossible causes:\n" +
"- Check 'ollama list' — make sure the model name matches exactly what's installed\n" +
"- The model may be corrupted. Try: ollama pull " + modelName + "\n" +
(useFreeformText ? "- No format constraint was sent — the model may not support freeform output.\n" : "") +
"- This Ollama version does not support format:json — using prompt instructions only (reliability varies)\n" +
"- If your model is very small (e.g., tinyllama, phi), try a larger one like llama3.1 or mistral"
);
@@ -512,6 +525,14 @@ class OllamaLlmProvider {
// ================================================================
// Step 5: Parse and return
// ================================================================
if (useFreeformText) {
return {
response: rawResponse,
providerApiPath: apiUsed,
providerExecution,
};
}
try {
return {
response: recoverJson(rawResponse),
@@ -553,17 +574,27 @@ class OpenAIReconstructionProvider {
{ signal, outputZodSchema } = {},
) {
if (!this.apiKey) throw new Error("OPENAI_API_KEY is not set");
const structuredOutputSchema = outputSchema
? createOpenAIStrictSchema(outputSchema, outputZodSchema ?? null)
: openAIReconstructionJsonSchema;
const format = {
type: "json_schema",
name: "reconstruction",
strict: true,
schema: structuredOutputSchema,
};
traceOpenAISchema({ model: modelName, format, schema: structuredOutputSchema });
// Explicit null → freeform text output (no JSON-schema constraint).
const useFreeformText = outputSchema === null;
let format;
if (useFreeformText) {
format = { type: "text" };
} else {
const structuredOutputSchema = outputSchema
? createOpenAIStrictSchema(outputSchema, outputZodSchema ?? null)
: openAIReconstructionJsonSchema;
format = {
type: "json_schema",
name: "reconstruction",
strict: true,
schema: structuredOutputSchema,
};
traceOpenAISchema({ model: modelName, format, schema: structuredOutputSchema });
}
const response = await this.fetchImpl("https://api.openai.com/v1/responses", {
method: "POST",
headers: {
@@ -599,6 +630,14 @@ class OpenAIReconstructionProvider {
throw error;
}
// Freeform mode — return raw text without JSON recovery or transformation.
if (useFreeformText) {
return {
response: outputText,
providerApiPath: "/v1/responses",
};
}
const recoveredResponse = recoverJson(outputText);
const normalisedResponse = outputSchema && !outputZodSchema
? recoveredResponse
tests/llm/provider.test.js
+133
View File
@@ -225,6 +225,139 @@ describe("OllamaLlmProvider chat capability detection", () => {
});
});
describe("Ollama freeform semantics", () => {
it("omits format constraint when outputSchema is explicitly null (chat path)", async () => {
const originalBaseUrl = process.env.OLLAMA_BASE_URL;
const fetchSpy = vi.fn()
.mockResolvedValueOnce({ ok: true, text: async () => "" })
.mockResolvedValueOnce({
ok: true,
json: async () => ({ message: { content: '{"freeform":"output"}' } }),
});
vi.stubGlobal("fetch", fetchSpy);
process.env.OLLAMA_BASE_URL = "http://ollama.test";
try {
__resetChatSupportForTests();
const provider = getProvider();
const result = await provider.generateReconstruction("prompt", "model", null, { signal: undefined });
expect(result.response).toBe("{\"freeform\":\"output\"}");
expect(result.providerApiPath).toBe("/api/chat");
// No format key in request body when schema is explicitly null
const requestBody = JSON.parse(fetchSpy.mock.calls[1][1].body);
expect(requestBody).not.toHaveProperty("format");
} finally {
vi.unstubAllGlobals();
if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
else process.env.OLLAMA_BASE_URL = originalBaseUrl;
}
});
it("omits format constraint when outputSchema is explicitly null (generate fallback)", async () => {
const originalBaseUrl = process.env.OLLAMA_BASE_URL;
const fetchSpy = vi.fn()
.mockResolvedValueOnce({ ok: false, status: 501, text: async () => "" })
.mockResolvedValueOnce({
ok: true,
json: async () => ({ response: "freeform text from /api/generate" }),
});
vi.stubGlobal("fetch", fetchSpy);
process.env.OLLAMA_BASE_URL = "http://ollama.test";
try {
__resetChatSupportForTests();
const provider = getProvider();
const result = await provider.generateReconstruction("prompt", "model", null, { signal: undefined });
expect(result.response).toBe("freeform text from /api/generate");
expect(result.providerApiPath).toBe("/api/generate");
// No format key in request body when schema is explicitly null
const requestBody = JSON.parse(fetchSpy.mock.calls[1][1].body);
expect(requestBody).not.toHaveProperty("format");
} finally {
vi.unstubAllGlobals();
if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
else process.env.OLLAMA_BASE_URL = originalBaseUrl;
}
});
it("still applies default schema when outputSchema is undefined", async () => {
const originalBaseUrl = process.env.OLLAMA_BASE_URL;
const fetchSpy = vi.fn()
.mockResolvedValueOnce({ ok: true, text: async () => "" })
.mockResolvedValueOnce({
ok: true,
json: async () => ({ message: { content: "{}" } }),
});
vi.stubGlobal("fetch", fetchSpy);
process.env.OLLAMA_BASE_URL = "http://ollama.test";
try {
__resetChatSupportForTests();
await getProvider().generateReconstruction("prompt", "model", undefined, { signal: undefined });
const requestBody = JSON.parse(fetchSpy.mock.calls[1][1].body);
// undefined → fallback to reconstructionJsonSchema (format IS present)
expect(requestBody.format).toBeDefined();
} finally {
vi.unstubAllGlobals();
if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
else process.env.OLLAMA_BASE_URL = originalBaseUrl;
}
});
});
describe("OpenAI freeform semantics", () => {
it("uses text format when outputSchema is explicitly null", async () => {
const provider = createOpenAIReconstructionProvider({
apiKey: "test-key",
fetchImpl: vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ output_text: "freeform text response" }),
}),
});
await provider.generateReconstruction("prompt", "model", null);
const request = JSON.parse(
provider.fetchImpl.mock.calls[0][1].body,
);
expect(request.text.format.type).toBe("text");
});
it("returns raw text without JSON recovery when outputSchema is explicitly null", async () => {
const provider = createOpenAIReconstructionProvider({
apiKey: "test-key",
fetchImpl: vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ output_text: "not-json-at-all {broken" }),
}),
});
const result = await provider.generateReconstruction("prompt", "model", null);
expect(result.response).toBe("not-json-at-all {broken");
expect(result.providerApiPath).toBe("/v1/responses");
});
it("still applies json_schema format when outputSchema is undefined", async () => {
const provider = createOpenAIReconstructionProvider({
apiKey: "test-key",
fetchImpl: vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ output_text: "{}" }),
}),
});
await provider.generateReconstruction("prompt");
const request = JSON.parse(provider.fetchImpl.mock.calls[0][1].body);
expect(request.text.format.type).toBe("json_schema");
});
});
describe("OpenAI reconstruction provider experiment seam", () => {
function assertStrictObjectInvariant(schema, root = schema, visited = new Set()) {
if (!schema || typeof schema !== "object" || visited.has(schema)) return;