import { z } from "zod"; import { reconstructionV2Schema } from "../reconstruction/schema.js"; import { isOpenAIUiJourneyExperiment, OPENAI_TERRA_MODEL } from "../config.js"; /** * Provider abstraction — the app calls getProvider() which returns an object * with a generateReconstruction(scenario, modelName) method. * Only Ollama is implemented right now; swapping providers requires only * changing getProvider(). */ export function getProvider() { if (isOpenAIUiJourneyExperiment()) return createOpenAIReconstructionProvider(); return new OllamaLlmProvider(); } /** Server-owned model resolution for the configured application provider. */ export function getProviderModelName() { return isOpenAIUiJourneyExperiment() ? OPENAI_TERRA_MODEL : process.env.OLLAMA_MODEL ?? null; } /** * Experiment-only construction seam. Production provider selection remains Ollama. * @param {{ apiKey?: string, fetchImpl?: typeof fetch }} [options] */ export function createOpenAIReconstructionProvider(options = {}) { return new OpenAIReconstructionProvider(options); } function recoverJson(raw) { if (typeof raw !== "string") return raw; const trimmed = raw.trim(); if (trimmed.length === 0) throw new SyntaxError("Model produced empty output"); try { return JSON.parse(trimmed); } catch { // Not directly parseable — try closing braces/brackets from the right side } let result = trimmed; let braceDepth = 0; let bracketDepth = 0; let inString = false; let escaped = false; for (let i = 0; i < result.length; i++) { const ch = result[i]; if (escaped) { escaped = false; continue; } if (ch === '\\') { escaped = true; continue; } if (ch === '"') { inString = !inString; continue; } if (inString) continue; if (ch === '{') braceDepth++; else if (ch === '}') braceDepth--; else if (ch === '[') bracketDepth++; else if (ch === ']') bracketDepth--; } const closingBrackets = []; for (let i = 0; i < bracketDepth; i++) closingBrackets.push(']'); for (let i = 0; i < braceDepth; i++) closingBrackets.push('}'); if (closingBrackets.length > 0) { const closed = result + closingBrackets.reverse().join(''); try { return JSON.parse(closed); } catch { /* still broken */ } } const lastOpen = Math.max(result.lastIndexOf('{'), result.lastIndexOf('[')); if (lastOpen >= 0) { try { return JSON.parse(result.slice(lastOpen)); } catch { /* nothing works */ } } throw new SyntaxError("Model output could not be parsed as JSON: " + result.slice(0, 300) + "..."); } function extractOpenAIResponseText(data) { if (typeof data.output_text === "string" && data.output_text.length > 0) { return data.output_text; } const outputText = (data.output ?? []).flatMap((item) => item?.type === "message" ? (item.content ?? []).flatMap((part) => part?.type === "output_text" && typeof part.text === "string" ? [part.text] : [], ) : [], ); if (outputText.length > 0) return outputText.join(""); const outputTypes = (data.output ?? []).map((item) => item?.type ?? "unknown"); const contentTypes = (data.output ?? []).flatMap((item) => (item?.content ?? []).map((part) => part?.type ?? "unknown"), ); const refusal = contentTypes.includes("refusal"); throw new Error( `OpenAI Responses API returned no output_text (output types: ${outputTypes.join(",") || "none"}; content types: ${contentTypes.join(",") || "none"}; refusal: ${refusal})`, ); } let _chatSupported = null; export const reconstructionJsonSchema = z.toJSONSchema(reconstructionV2Schema); const openAIReconstructionJsonSchema = createOpenAIStrictSchema( reconstructionJsonSchema, reconstructionV2Schema, ); /** @internal OpenAI Structured Outputs requires every object property. */ export function createOpenAIStrictSchema(schema, zodSchema = reconstructionV2Schema) { const projected = structuredClone(schema); projectOpenAIStrictSchema(projected, zodSchema, projected, new WeakSet()); return projected; } /** @internal Remove OpenAI null placeholders for canonically optional fields only. */ export function normaliseOpenAITransportResponse( value, zodSchema = reconstructionV2Schema, schema = reconstructionJsonSchema, ) { return normaliseTransportValue(value, schema, zodSchema, schema); } function resolveSchema(schema, rootSchema) { if (!schema?.$ref) return schema; const path = schema.$ref.replace(/^#\//, "").split("/"); return path.reduce((value, key) => value?.[key], rootSchema) ?? schema; } function zodDef(schema) { return schema?._zod?.def ?? schema?._def; } function unwrapZodSchema(schema) { const def = zodDef(schema); if (["optional", "nullable", "default"].includes(def?.type)) { return unwrapZodSchema(def.innerType); } return schema; } function zodObjectShape(schema) { const def = zodDef(unwrapZodSchema(schema)); return def?.type === "object" ? def.shape : null; } function zodArrayItem(schema) { const def = zodDef(unwrapZodSchema(schema)); return def?.type === "array" ? def.element : null; } function zodAcceptsNull(schema) { return schema?.isNullable?.() === true; } function projectOpenAIStrictSchema(schema, zodSchema, rootSchema, visited) { const resolved = resolveSchema(schema, rootSchema); if (!resolved || typeof resolved !== "object" || visited.has(resolved)) return; visited.add(resolved); const shape = zodObjectShape(zodSchema); if (resolved?.type === "object" || resolved?.properties) { if (shape) { for (const [key, property] of Object.entries(resolved.properties)) { const propertyZodSchema = shape[key]; if (propertyZodSchema?.isOptional?.() && !schemaAllowsNull(property, rootSchema)) { resolved.properties[key] = { anyOf: [property, { type: "null" }] }; } } } resolved.additionalProperties = false; } if (resolved?.items) { projectOpenAIStrictSchema(resolved.items, zodArrayItem(zodSchema), rootSchema, visited); } if (resolved?.properties) { for (const [key, property] of Object.entries(resolved.properties)) { projectOpenAIStrictSchema(property, shape?.[key], rootSchema, visited); } resolved.required = Object.keys(resolved.properties); } for (const branch of [ ...(resolved.anyOf ?? []), ...(resolved.oneOf ?? []), ...(resolved.allOf ?? []), ]) { projectOpenAIStrictSchema(branch, null, rootSchema, visited); } for (const definition of Object.values(resolved.$defs ?? resolved.definitions ?? {})) { projectOpenAIStrictSchema(definition, null, rootSchema, visited); } } function satisfiesOpenAIStrictSchema(schema, visited = new WeakSet()) { if (!schema || typeof schema !== "object" || visited.has(schema)) return true; visited.add(schema); const isObject = schema.type === "object" || schema.properties; if (isObject && schema.additionalProperties !== false) return false; if (schema.properties) { const propertyKeys = Object.keys(schema.properties); if ( !Array.isArray(schema.required) || schema.required.length !== propertyKeys.length || !propertyKeys.every((key) => schema.required.includes(key)) ) return false; } return [ schema.items, ...Object.values(schema.properties ?? {}), ...(schema.anyOf ?? []), ...(schema.oneOf ?? []), ...(schema.allOf ?? []), ...Object.values(schema.$defs ?? schema.definitions ?? {}), ].every((child) => satisfiesOpenAIStrictSchema(child, visited)); } function traceOpenAISchema({ model, format, schema }) { if (process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA !== "1") return; const serializedSchema = JSON.stringify(schema); const finalSchema = JSON.parse(serializedSchema); const properties = finalSchema.properties ?? {}; const required = finalSchema.required ?? []; const relationships = properties.relationships; console.info("[confidence-engine][openai-schema-trace]", { model, formatType: format.type, formatName: format.name, strict: format.strict, rootProperties: Object.keys(properties), rootRequired: required, rootSetsEqual: required.length === Object.keys(properties).length && Object.keys(properties).every((key) => required.includes(key)), relationshipsInProperties: Object.prototype.hasOwnProperty.call(properties, "relationships"), relationshipsInRequired: required.includes("relationships"), relationshipsItemType: relationships?.items?.type ?? null, relationshipsItemAdditionalProperties: relationships?.items?.additionalProperties ?? null, rootAdditionalProperties: finalSchema.additionalProperties ?? null, recursiveStrictInvariant: satisfiesOpenAIStrictSchema(finalSchema), }); } function schemaAllowsNull(schema, rootSchema) { const resolved = resolveSchema(schema, rootSchema); return ( resolved?.type === "null" || (Array.isArray(resolved?.type) && resolved.type.includes("null")) || [...(resolved?.anyOf ?? []), ...(resolved?.oneOf ?? [])].some((branch) => schemaAllowsNull(branch, rootSchema), ) ); } function normaliseTransportValue(value, schema, zodSchema, rootSchema) { const resolved = resolveSchema(schema, rootSchema); if (Array.isArray(value) && resolved?.items) { return value.map((item) => normaliseTransportValue(item, resolved.items, zodArrayItem(zodSchema), rootSchema), ); } if (!value || typeof value !== "object" || !resolved?.properties) return value; const shape = zodObjectShape(zodSchema); const normalised = {}; for (const [key, item] of Object.entries(value)) { const propertySchema = resolved.properties[key]; const propertyZodSchema = shape?.[key]; if (!propertySchema || !propertyZodSchema) { normalised[key] = item; } else if (item === null && propertyZodSchema.isOptional?.() && !zodAcceptsNull(propertyZodSchema)) { continue; } else { normalised[key] = normaliseTransportValue( item, propertySchema, propertyZodSchema, rootSchema, ); } } return normalised; } /** @internal Test-only seam for isolated provider capability scenarios. */ export function __resetChatSupportForTests() { _chatSupported = null; } async function detectChatSupport(baseUrl, modelName) { if (_chatSupported !== null) return _chatSupported; try { const res = await fetch(`${baseUrl}/api/chat`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model: modelName, messages: [{ role: "user", content: "test" }], stream: false, }), }); if (res.ok) { await res.text(); _chatSupported = true; } else if (res.status === 405 || res.status === 501) { await res.text(); _chatSupported = false; } else { await res.text(); _chatSupported = false; } } catch { _chatSupported = false; } return _chatSupported; } class OllamaLlmProvider { async generateReconstruction(scenario, modelName, outputSchema) { // scenario is ALREADY a fully-built prompt text (built by analyseScenario). // Do NOT call buildPrompt() again — that would double-wrap the prompt. const prompt = scenario; const baseUrl = process.env.OLLAMA_BASE_URL; if (!baseUrl) throw new Error("OLLAMA_BASE_URL is not set"); let apiUsed = null; let chatSupported = false; let rawResponse = null; let fullResponseData = null; const providerExecution = { chatCapabilityDetected: false, chatRequestAttempted: false, chatRequestSucceeded: false, generateRequestAttempted: false, }; // ================================================================ // Step 1: Detect whether /api/chat exists (cache result) // ================================================================ try { chatSupported = await detectChatSupport(baseUrl, modelName); } catch { /* failed silently — defaults to false */ } providerExecution.chatCapabilityDetected = chatSupported; // ================================================================ // Step 2: Try /api/chat if supported with the supplied or reconstruction schema // ================================================================ if (chatSupported) { try { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 300000); // 5 min for reconstruction apiUsed = "/api/chat"; providerExecution.chatRequestAttempted = true; const res = await fetch(`${baseUrl}/api/chat`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model: modelName, messages: [{ role: "user", content: prompt }], stream: false, format: outputSchema ?? reconstructionJsonSchema, }), signal: controller.signal, }); clearTimeout(timeout); if (res.ok) { providerExecution.chatRequestSucceeded = true; fullResponseData = await res.json(); rawResponse = typeof fullResponseData.message?.content === "string" ? fullResponseData.message.content : JSON.stringify(fullResponseData.message?.content ?? null); apiUsed = "/api/chat"; } else { await res.text(); } } catch (e) { if (!e.message.includes("abort")) { /* non-fatal */ } } } // ================================================================ // Step 3: /api/generate (works on all Ollama versions) // Use a long timeout — cold starts can take 2-4 minutes for large models. // ================================================================ if (rawResponse == null || rawResponse === "") { try { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 300000); // 5 min for cold start apiUsed = "/api/generate"; // set BEFORE the request so we know which API failed providerExecution.generateRequestAttempted = true; const res = await fetch(`${baseUrl}/api/generate`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model: modelName, prompt, stream: false, // No format:json — older Ollama doesn't support it on /api/generate either. // We rely on the strong prompt instruction above instead. }), signal: controller.signal, }); clearTimeout(timeout); if (!res.ok) { const text = await res.text().catch(() => ""); throw new Error(`/api/generate returned ${res.status}: ${text.slice(0, 500)}`); } fullResponseData = await res.json(); rawResponse = typeof fullResponseData.response === "string" ? fullResponseData.response : JSON.stringify(fullResponseData); } catch (e) { if (apiUsed === "/api/generate") { const error = new Error( `Ollama /api/generate request timed out after 5 minutes.\n\n` + `This usually means:\n` + `1. The model is loading into memory for the first time (cold start) — this can take several minutes\n` + `2. Your hardware is slow for this model size\n` + `3. Ollama server is overloaded\n\n` + `Try:\n` + `- Run the request again after ~1 minute (model may be cached now)\n` + `- Use a smaller model (e.g., llama3.1 instead of llama3.1:70b)\n` + `- Check Ollama logs: \`ollama serve\` or look at your system logs` ); if (e?.name === "AbortError" || e instanceof TypeError) { error.code = "PROVIDER_UNAVAILABLE"; } error.providerApiPath = apiUsed; error.providerExecution = providerExecution; throw error; } throw e; } } // ================================================================ // Step 4: Diagnose empty output // ================================================================ if (rawResponse === "") { let diagInfo = ""; if (fullResponseData) { const keys = Object.keys(fullResponseData); diagInfo = "Keys in response: " + keys.join(", ") + "\n"; for (const key of keys) { const val = fullResponseData[key]; if (typeof val === "string") { diagInfo += ` ${key}: "${val.slice(0, 200)}"\n`; } else if (typeof val === "object" && val != null) { try { diagInfo += ` ${key}: ${JSON.stringify(val).slice(0, 300)}\n`; } catch { diagInfo += ` ${key}: [object]\n`; } } else { diagInfo += ` ${key}: ${String(val)}\n`; } } } const error = new Error( "Model produced empty output.\n\n" + "API used: " + (apiUsed || "none") + "\n" + "/api/chat supported: " + chatSupported + "\n" + "Full server response:\n" + (diagInfo || "(none)\n") + "\nPossible causes:\n" + "- Check 'ollama list' — make sure the model name matches exactly what's installed\n" + "- The model may be corrupted. Try: ollama pull " + modelName + "\n" + "- This Ollama version does not support format:json — using prompt instructions only (reliability varies)\n" + "- If your model is very small (e.g., tinyllama, phi), try a larger one like llama3.1 or mistral" ); error.providerApiPath = apiUsed; error.providerExecution = providerExecution; throw error; } // ================================================================ // Step 5: Parse and return // ================================================================ try { return { response: recoverJson(rawResponse), providerApiPath: apiUsed, providerExecution, }; } catch (e) { if (e instanceof SyntaxError) { const error = new Error( "Model returned output that could not be parsed as valid JSON.\n\n" + "API used: " + (apiUsed || "none") + "\n" + "/api/chat supported: " + chatSupported + "\n" + "Raw model output:\n" + rawResponse.slice(0, 1000) + (rawResponse.length > 1000 ? "\n...(truncated)" : "") + "\n\nPossible causes:\n" + "- This Ollama version does not support format:json. The model is producing free-form text.\n" + "- Try a larger model (llama3.1, mistral-large) which follows JSON instructions better\n" + "- Shorten your scenario to under 500 words\n" + "- Consider upgrading Ollama: https://ollama.com/download" ); error.providerApiPath = apiUsed; error.providerExecution = providerExecution; throw error; } throw e; } } } class OpenAIReconstructionProvider { constructor({ apiKey = process.env.OPENAI_API_KEY, fetchImpl = fetch } = {}) { this.apiKey = apiKey; this.fetchImpl = fetchImpl; } async generateReconstruction(prompt, modelName = "gpt-5.6-terra", outputSchema) { if (!this.apiKey) throw new Error("OPENAI_API_KEY is not set"); const structuredOutputSchema = outputSchema ? createOpenAIStrictSchema(outputSchema, null) : openAIReconstructionJsonSchema; const format = { type: "json_schema", name: "reconstruction", strict: true, schema: structuredOutputSchema, }; traceOpenAISchema({ model: modelName, format, schema: structuredOutputSchema }); const response = await this.fetchImpl("https://api.openai.com/v1/responses", { method: "POST", headers: { Authorization: `Bearer ${this.apiKey}`, "Content-Type": "application/json", }, body: JSON.stringify({ model: modelName, input: prompt, text: { format, }, }), }); if (!response.ok) { const body = await response.text(); const error = new Error( `OpenAI Responses API returned ${response.status}: ${body}`, ); error.providerApiPath = "/v1/responses"; throw error; } const data = await response.json(); let outputText; try { outputText = extractOpenAIResponseText(data); } catch (cause) { const error = new Error(cause.message); error.providerApiPath = "/v1/responses"; throw error; } return { response: outputSchema ? recoverJson(outputText) : normaliseOpenAITransportResponse(recoverJson(outputText)), providerApiPath: "/v1/responses", }; } }