experiment(confidence-engine): add openai provider apparatus

This commit is contained in:
2026-09-06 07:50:47 +01:00
parent 653934559c
commit 860ee6fc5b
3 changed files with 133 additions and 1 deletions
+6
View File
@@ -116,6 +116,12 @@
- This increment adds only a final semantic-preservation self-check, testing instruction salience rather than missing semantic specification. No live model calls were made.
- Next boundary: one manual fixed-scenario Postman production run, scoring C first while checking A/B/D1/D2/E and unsupported expansion or causal strengthening for regressions.
## OpenAI reconstruction-provider experiment apparatus
- Structural Ollama/provider issues were previously fixed; current semantic comparison now warrants testing model-compliance variance.
- An experiment-only OpenAI Responses API provider apparatus exists for `gpt-5.6-terra`, using the same current v0.5 prompt and canonical reconstruction schema. Production provider selection remains Ollama/Qwen.
- No live OpenAI calls have occurred. Next boundary: a bounded live semantic comparison, not production migration.
## Current product architecture
Three distinct routes, not a single page:
+61
View File
@@ -12,6 +12,14 @@ export function getProvider() {
return new OllamaLlmProvider();
}
/**
* Experiment-only construction seam. Production provider selection remains Ollama.
* @param {{ apiKey?: string, fetchImpl?: typeof fetch }} [options]
*/
export function createOpenAIReconstructionProvider(options = {}) {
return new OpenAIReconstructionProvider(options);
}
function recoverJson(raw) {
if (typeof raw !== "string") return raw;
const trimmed = raw.trim();
@@ -294,3 +302,56 @@ class OllamaLlmProvider {
}
}
}
class OpenAIReconstructionProvider {
constructor({ apiKey = process.env.OPENAI_API_KEY, fetchImpl = fetch } = {}) {
this.apiKey = apiKey;
this.fetchImpl = fetchImpl;
}
async generateReconstruction(prompt, modelName = "gpt-5.6-terra") {
if (!this.apiKey) throw new Error("OPENAI_API_KEY is not set");
const response = await this.fetchImpl("https://api.openai.com/v1/responses", {
method: "POST",
headers: {
Authorization: `Bearer ${this.apiKey}`,
"Content-Type": "application/json",
},
body: JSON.stringify({
model: modelName,
input: prompt,
text: {
format: {
type: "json_schema",
name: "reconstruction",
strict: true,
schema: reconstructionJsonSchema,
},
},
}),
});
if (!response.ok) {
const body = await response.text();
const error = new Error(
`OpenAI Responses API returned ${response.status}: ${body}`,
);
error.providerApiPath = "/v1/responses";
throw error;
}
const data = await response.json();
const outputText = data.output_text;
if (typeof outputText !== "string") {
const error = new Error("OpenAI Responses API returned no output_text");
error.providerApiPath = "/v1/responses";
throw error;
}
return {
response: recoverJson(outputText),
providerApiPath: "/v1/responses",
};
}
}
+66 -1
View File
@@ -1,5 +1,9 @@
import { describe, expect, it, vi } from "vitest";
import { __resetChatSupportForTests, getProvider } from "@/lib/llm/provider.js";
import {
__resetChatSupportForTests,
createOpenAIReconstructionProvider,
getProvider,
} from "@/lib/llm/provider.js";
describe("OllamaLlmProvider chat capability detection", () => {
it("uses the configured model for the chat probe and keeps the chat path", async () => {
@@ -115,3 +119,64 @@ describe("OllamaLlmProvider chat capability detection", () => {
}
});
});
describe("OpenAI reconstruction provider experiment seam", () => {
it("uses the Responses API with the canonical strict reconstruction schema", async () => {
const fetchSpy = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ output_text: '{"reconstruction":"result"}' }),
});
const provider = createOpenAIReconstructionProvider({
apiKey: "test-key",
fetchImpl: fetchSpy,
});
const result = await provider.generateReconstruction(
"current reconstruction prompt",
);
const request = JSON.parse(fetchSpy.mock.calls[0][1].body);
const schemaText = JSON.stringify(request.text.format.schema);
expect(fetchSpy).toHaveBeenCalledWith(
"https://api.openai.com/v1/responses",
expect.objectContaining({ method: "POST" }),
);
expect(request).toMatchObject({
model: "gpt-5.6-terra",
input: "current reconstruction prompt",
text: {
format: {
type: "json_schema",
name: "reconstruction",
strict: true,
},
},
});
expect(schemaText).toContain("relationship");
expect(schemaText).toContain("evidenceType");
expect(schemaText).toContain("primaryType");
expect(schemaText).toContain("secondaryTypes");
expect(schemaText).toContain("confidence");
expect(schemaText).toContain("importance");
expect(result).toEqual({
response: { reconstruction: "result" },
providerApiPath: "/v1/responses",
});
});
it("surfaces Responses API failures without inventing reconstruction content", async () => {
const provider = createOpenAIReconstructionProvider({
apiKey: "test-key",
fetchImpl: vi.fn().mockResolvedValue({
ok: false,
status: 429,
text: async () => "rate limited",
}),
});
await expect(provider.generateReconstruction("prompt")).rejects.toMatchObject({
providerApiPath: "/v1/responses",
message: "OpenAI Responses API returned 429: rate limited",
});
});
});