experiment(confidence-engine): complete openai reconstruction apparatus

lib/llm/provider.js

@@ -71,6 +71,122 @@ function recoverJson(raw) {
 
 let _chatSupported = null;
 const reconstructionJsonSchema = z.toJSONSchema(reconstructionV2Schema);
+const openAIReconstructionJsonSchema = createOpenAIStrictSchema(
+  reconstructionJsonSchema,
+  reconstructionV2Schema,
+);
+
+/** @internal OpenAI Structured Outputs requires every object property. */
+export function createOpenAIStrictSchema(schema, zodSchema = reconstructionV2Schema) {
+  const projected = structuredClone(schema);
+  projectOpenAIStrictSchema(projected, zodSchema, projected);
+  return projected;
+}
+
+/** @internal Remove OpenAI null placeholders for canonically optional fields only. */
+export function normaliseOpenAITransportResponse(
+  value,
+  zodSchema = reconstructionV2Schema,
+  schema = reconstructionJsonSchema,
+) {
+  return normaliseTransportValue(value, schema, zodSchema, schema);
+}
+
+function resolveSchema(schema, rootSchema) {
+  if (!schema?.$ref) return schema;
+  const path = schema.$ref.replace(/^#\//, "").split("/");
+  return path.reduce((value, key) => value?.[key], rootSchema) ?? schema;
+}
+
+function zodDef(schema) {
+  return schema?._zod?.def ?? schema?._def;
+}
+
+function unwrapZodSchema(schema) {
+  const def = zodDef(schema);
+  if (["optional", "nullable", "default"].includes(def?.type)) {
+    return unwrapZodSchema(def.innerType);
+  }
+  return schema;
+}
+
+function zodObjectShape(schema) {
+  const def = zodDef(unwrapZodSchema(schema));
+  return def?.type === "object" ? def.shape : null;
+}
+
+function zodArrayItem(schema) {
+  const def = zodDef(unwrapZodSchema(schema));
+  return def?.type === "array" ? def.element : null;
+}
+
+function zodAcceptsNull(schema) {
+  return schema?.isNullable?.() === true;
+}
+
+function projectOpenAIStrictSchema(schema, zodSchema, rootSchema) {
+  const resolved = resolveSchema(schema, rootSchema);
+  const shape = zodObjectShape(zodSchema);
+  if (resolved?.properties && shape) {
+    for (const [key, property] of Object.entries(resolved.properties)) {
+      const propertyZodSchema = shape[key];
+      if (propertyZodSchema?.isOptional?.() && !schemaAllowsNull(property, rootSchema)) {
+        resolved.properties[key] = { anyOf: [property, { type: "null" }] };
+      }
+    }
+    resolved.required = Object.keys(resolved.properties);
+  }
+
+  if (resolved?.items) {
+    projectOpenAIStrictSchema(resolved.items, zodArrayItem(zodSchema), rootSchema);
+  }
+  if (resolved?.properties && shape) {
+    for (const [key, property] of Object.entries(resolved.properties)) {
+      projectOpenAIStrictSchema(property, shape[key], rootSchema);
+    }
+  }
+}
+
+function schemaAllowsNull(schema, rootSchema) {
+  const resolved = resolveSchema(schema, rootSchema);
+  return (
+    resolved?.type === "null" ||
+    (Array.isArray(resolved?.type) && resolved.type.includes("null")) ||
+    [...(resolved?.anyOf ?? []), ...(resolved?.oneOf ?? [])].some((branch) =>
+      schemaAllowsNull(branch, rootSchema),
+    )
+  );
+}
+
+function normaliseTransportValue(value, schema, zodSchema, rootSchema) {
+  const resolved = resolveSchema(schema, rootSchema);
+  if (Array.isArray(value) && resolved?.items) {
+    return value.map((item) =>
+      normaliseTransportValue(item, resolved.items, zodArrayItem(zodSchema), rootSchema),
+    );
+  }
+  if (!value || typeof value !== "object" || !resolved?.properties) return value;
+
+  const shape = zodObjectShape(zodSchema);
+  const normalised = {};
+  for (const [key, item] of Object.entries(value)) {
+    const propertySchema = resolved.properties[key];
+    const propertyZodSchema = shape?.[key];
+    if (!propertySchema || !propertyZodSchema) {
+      normalised[key] = item;
+    } else if (item === null && propertyZodSchema.isOptional?.() && !zodAcceptsNull(propertyZodSchema)) {
+      continue;
+    } else {
+      normalised[key] = normaliseTransportValue(
+        item,
+        propertySchema,
+        propertyZodSchema,
+        rootSchema,
+      );
+    }
+  }
+  return normalised;
+}
 
 /** @internal Test-only seam for isolated provider capability scenarios. */
 export function __resetChatSupportForTests() {
@@ -326,7 +442,7 @@ class OpenAIReconstructionProvider {
             type: "json_schema",
             name: "reconstruction",
             strict: true,
-            schema: reconstructionJsonSchema,
+            schema: openAIReconstructionJsonSchema,
           },
         },
       }),
@@ -350,7 +466,7 @@ class OpenAIReconstructionProvider {
     }
 
     return {
-      response: recoverJson(outputText),
+      response: normaliseOpenAITransportResponse(recoverJson(outputText)),
       providerApiPath: "/v1/responses",
     };
   }