From 215c783d11581802ff1b01b649e91442e3bd8735 Mon Sep 17 00:00:00 2001 From: robbond Date: Sun, 6 Sep 2026 10:06:06 +0100 Subject: [PATCH] experiment(confidence-engine): complete openai reconstruction apparatus --- docs/current-handoff.md | 11 ++ lib/llm/provider.js | 120 +++++++++++++++++- scripts/start-case-experiment-helper.cjs | 24 +++- tests/llm/provider.test.js | 75 +++++++++++ .../start-case-experiment-helper.test.js | 58 ++++++++- 5 files changed, 281 insertions(+), 7 deletions(-) diff --git a/docs/current-handoff.md b/docs/current-handoff.md index cd6a766..6de96b0 100644 --- a/docs/current-handoff.md +++ b/docs/current-handoff.md @@ -134,6 +134,17 @@ - Its import-only proof succeeded with `startCaseResolved: true`: the real `startCase()` dependency chain resolves and no provider call occurs. Production runtime and the Ollama/Qwen default remain unchanged. - Next boundary: exactly one live GPT-5.6 Terra reconstruction through this canonical helper. +## OpenAI CLI experiment selection + +- The canonical helper now exposes experiment-only OpenAI selection through `START_CASE_EXPERIMENT_PROVIDER=openai`, reusing the existing OpenAI provider constructor and provider-injection seam. Its default behavior is unchanged. +- Import-only mode remains inert, production provider selection remains Ollama/Qwen, and no live calls occurred. Next boundary: exactly one live OpenAI call using the fixed manufacturing scenario. + +## OpenAI strict reconstruction transport apparatus + +- The first real OpenAI request reached Responses API but failed before inference with `invalid_json_schema`; the first reported incompatibility was optional `unexplainedTransitions[].entity` absent from `required`. +- OpenAI strict transport projection now derives structure from canonical Zod-generated JSON Schema and canonical input optionality from the Zod contract, including `.optional().default(...)`. Canonically omittable fields are required-but-nullable for transport; null placeholders are omitted only for those fields before canonical Zod validation, while genuinely required fields remain non-nullable. +- `reconstructionV2Schema` and Ollama behavior remain unchanged. `START_CASE_EXPERIMENT_PROVIDER=openai` is experiment-only selection; default behavior remains Ollama/Qwen. Deterministic closeout passed with zero live calls; next boundary is exactly one live GPT-5.6 Terra reconstruction. + ## Current product architecture Three distinct routes, not a single page: diff --git a/lib/llm/provider.js b/lib/llm/provider.js index aec4778..6fc6489 100644 --- a/lib/llm/provider.js +++ b/lib/llm/provider.js @@ -71,6 +71,122 @@ function recoverJson(raw) { let _chatSupported = null; const reconstructionJsonSchema = z.toJSONSchema(reconstructionV2Schema); +const openAIReconstructionJsonSchema = createOpenAIStrictSchema( + reconstructionJsonSchema, + reconstructionV2Schema, +); + +/** @internal OpenAI Structured Outputs requires every object property. */ +export function createOpenAIStrictSchema(schema, zodSchema = reconstructionV2Schema) { + const projected = structuredClone(schema); + projectOpenAIStrictSchema(projected, zodSchema, projected); + return projected; +} + +/** @internal Remove OpenAI null placeholders for canonically optional fields only. */ +export function normaliseOpenAITransportResponse( + value, + zodSchema = reconstructionV2Schema, + schema = reconstructionJsonSchema, +) { + return normaliseTransportValue(value, schema, zodSchema, schema); +} + +function resolveSchema(schema, rootSchema) { + if (!schema?.$ref) return schema; + const path = schema.$ref.replace(/^#\//, "").split("/"); + return path.reduce((value, key) => value?.[key], rootSchema) ?? schema; +} + +function zodDef(schema) { + return schema?._zod?.def ?? schema?._def; +} + +function unwrapZodSchema(schema) { + const def = zodDef(schema); + if (["optional", "nullable", "default"].includes(def?.type)) { + return unwrapZodSchema(def.innerType); + } + return schema; +} + +function zodObjectShape(schema) { + const def = zodDef(unwrapZodSchema(schema)); + return def?.type === "object" ? def.shape : null; +} + +function zodArrayItem(schema) { + const def = zodDef(unwrapZodSchema(schema)); + return def?.type === "array" ? def.element : null; +} + +function zodAcceptsNull(schema) { + return schema?.isNullable?.() === true; +} + +function projectOpenAIStrictSchema(schema, zodSchema, rootSchema) { + const resolved = resolveSchema(schema, rootSchema); + const shape = zodObjectShape(zodSchema); + if (resolved?.properties && shape) { + for (const [key, property] of Object.entries(resolved.properties)) { + const propertyZodSchema = shape[key]; + if (propertyZodSchema?.isOptional?.() && !schemaAllowsNull(property, rootSchema)) { + resolved.properties[key] = { anyOf: [property, { type: "null" }] }; + } + } + resolved.required = Object.keys(resolved.properties); + } + + if (resolved?.items) { + projectOpenAIStrictSchema(resolved.items, zodArrayItem(zodSchema), rootSchema); + } + if (resolved?.properties && shape) { + for (const [key, property] of Object.entries(resolved.properties)) { + projectOpenAIStrictSchema(property, shape[key], rootSchema); + } + } +} + +function schemaAllowsNull(schema, rootSchema) { + const resolved = resolveSchema(schema, rootSchema); + return ( + resolved?.type === "null" || + (Array.isArray(resolved?.type) && resolved.type.includes("null")) || + [...(resolved?.anyOf ?? []), ...(resolved?.oneOf ?? [])].some((branch) => + schemaAllowsNull(branch, rootSchema), + ) + ); +} + +function normaliseTransportValue(value, schema, zodSchema, rootSchema) { + const resolved = resolveSchema(schema, rootSchema); + if (Array.isArray(value) && resolved?.items) { + return value.map((item) => + normaliseTransportValue(item, resolved.items, zodArrayItem(zodSchema), rootSchema), + ); + } + if (!value || typeof value !== "object" || !resolved?.properties) return value; + + const shape = zodObjectShape(zodSchema); + const normalised = {}; + for (const [key, item] of Object.entries(value)) { + const propertySchema = resolved.properties[key]; + const propertyZodSchema = shape?.[key]; + if (!propertySchema || !propertyZodSchema) { + normalised[key] = item; + } else if (item === null && propertyZodSchema.isOptional?.() && !zodAcceptsNull(propertyZodSchema)) { + continue; + } else { + normalised[key] = normaliseTransportValue( + item, + propertySchema, + propertyZodSchema, + rootSchema, + ); + } + } + return normalised; +} /** @internal Test-only seam for isolated provider capability scenarios. */ export function __resetChatSupportForTests() { @@ -326,7 +442,7 @@ class OpenAIReconstructionProvider { type: "json_schema", name: "reconstruction", strict: true, - schema: reconstructionJsonSchema, + schema: openAIReconstructionJsonSchema, }, }, }), @@ -350,7 +466,7 @@ class OpenAIReconstructionProvider { } return { - response: recoverJson(outputText), + response: normaliseOpenAITransportResponse(recoverJson(outputText)), providerApiPath: "/v1/responses", }; } diff --git a/scripts/start-case-experiment-helper.cjs b/scripts/start-case-experiment-helper.cjs index b3ce8ab..82556df 100755 --- a/scripts/start-case-experiment-helper.cjs +++ b/scripts/start-case-experiment-helper.cjs @@ -62,8 +62,26 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti } let startCase = options.startCase; + let reconstructionProvider = options.reconstructionProvider; + let reconstructionModelName = options.reconstructionModelName; const isMock = process.env.START_CASE_EXPERIMENT_HELPER_MOCK === "1"; + if (process.env.START_CASE_EXPERIMENT_PROVIDER === "openai") { + if (!process.env.OPENAI_API_KEY) { + throw new Error( + "START_CASE_EXPERIMENT_PROVIDER=openai requires OPENAI_API_KEY", + ); + } + const { createOpenAIReconstructionProvider } = await import( + PROJECT_ROOT + "/lib/llm/provider.js" + ); + reconstructionProvider = createOpenAIReconstructionProvider({ + apiKey: process.env.OPENAI_API_KEY, + fetchImpl: fetch, + }); + reconstructionModelName = "gpt-5.6-terra"; + } + if (!startCase && isMock) { // Deterministic mode: skip environment checks and use inline test double. startCase = async (body) => { @@ -100,8 +118,8 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti const endToEndStartedAt = Date.now(); try { result = await startCase(scenarioInput, { - reconstructionProvider: options.reconstructionProvider, - reconstructionModelName: options.reconstructionModelName, + reconstructionProvider, + reconstructionModelName, }); } finally { // Clean up experiment env var after execution regardless of outcome @@ -191,5 +209,3 @@ if (require.main === module) { } module.exports = { runStartCaseExperiment }; - -module.exports = { runStartCaseExperiment }; diff --git a/tests/llm/provider.test.js b/tests/llm/provider.test.js index 7a79675..0d244bc 100644 --- a/tests/llm/provider.test.js +++ b/tests/llm/provider.test.js @@ -1,9 +1,13 @@ import { describe, expect, it, vi } from "vitest"; import { __resetChatSupportForTests, + createOpenAIStrictSchema, createOpenAIReconstructionProvider, getProvider, + normaliseOpenAITransportResponse, } from "@/lib/llm/provider.js"; +import { reconstructionV2Schema } from "@/lib/reconstruction/schema.js"; +import { z } from "zod"; describe("OllamaLlmProvider chat capability detection", () => { it("uses the configured model for the chat probe and keeps the chat path", async () => { @@ -121,6 +125,76 @@ describe("OllamaLlmProvider chat capability detection", () => { }); describe("OpenAI reconstruction provider experiment seam", () => { + it("projects canonical optional fields as required but nullable", () => { + const nativeSchema = z.toJSONSchema(reconstructionV2Schema); + const projectedSchema = createOpenAIStrictSchema( + nativeSchema, + reconstructionV2Schema, + ); + + function resolveLocalRef(schema, root = projectedSchema) { + if (!schema.$ref) return schema; + return schema.$ref + .replace(/^#\//, "") + .split("/") + .reduce((value, key) => value?.[key], root); + } + + function acceptsNull(schema, root = projectedSchema) { + const resolved = resolveLocalRef(schema, root); + return ( + resolved?.type === "null" || + (Array.isArray(resolved?.type) && resolved.type.includes("null")) || + [...(resolved?.anyOf ?? []), ...(resolved?.oneOf ?? [])].some((branch) => + acceptsNull(branch, root), + ) + ); + } + + function assertAllPropertiesRequired(schema, root = projectedSchema) { + const resolved = resolveLocalRef(schema, root); + if (resolved?.properties) { + expect(resolved.required).toEqual(Object.keys(resolved.properties)); + Object.values(resolved.properties).forEach((property) => assertAllPropertiesRequired(property, root)); + } + (resolved?.anyOf ?? []).forEach((branch) => assertAllPropertiesRequired(branch, root)); + (resolved?.oneOf ?? []).forEach((branch) => assertAllPropertiesRequired(branch, root)); + if (resolved?.items) assertAllPropertiesRequired(resolved.items, root); + } + + assertAllPropertiesRequired(projectedSchema); + const transition = projectedSchema.properties.reconstruction.properties.unexplainedTransitions.items; + expect(transition.required).toContain("entity"); + expect(acceptsNull(transition.properties.entity)).toBe(true); + const inputClassification = resolveLocalRef( + projectedSchema.properties.inputClassification, + ); + expect(inputClassification.required).toContain("secondaryTypes"); + expect(acceptsNull(inputClassification.properties.secondaryTypes)).toBe(true); + expect(acceptsNull(transition.properties.id)).toBe(false); + }); + + it("removes only optional transport null placeholders", () => { + const schema = z.object({ + optionalText: z.string().optional(), + nullableText: z.string().nullable(), + requiredText: z.string(), + children: z.array(z.object({ optionalChild: z.string().optional() })), + }); + const nativeSchema = z.toJSONSchema(schema); + + expect(normaliseOpenAITransportResponse({ + optionalText: null, + nullableText: null, + requiredText: null, + children: [{ optionalChild: null }], + }, schema, nativeSchema)).toEqual({ + nullableText: null, + requiredText: null, + children: [{}], + }); + }); + it("uses the Responses API with the canonical strict reconstruction schema", async () => { const fetchSpy = vi.fn().mockResolvedValue({ ok: true, @@ -158,6 +232,7 @@ describe("OpenAI reconstruction provider experiment seam", () => { expect(schemaText).toContain("secondaryTypes"); expect(schemaText).toContain("confidence"); expect(schemaText).toContain("importance"); + expect(request.text.format.schema).not.toEqual(z.toJSONSchema(reconstructionV2Schema)); expect(result).toEqual({ response: { reconstruction: "result" }, providerApiPath: "/v1/responses", diff --git a/tests/scripts/start-case-experiment-helper.test.js b/tests/scripts/start-case-experiment-helper.test.js index 9e35ebe..21d934a 100644 --- a/tests/scripts/start-case-experiment-helper.test.js +++ b/tests/scripts/start-case-experiment-helper.test.js @@ -9,7 +9,7 @@ * with a deterministic inline double (zero live model calls). */ -import { describe, expect, it, beforeAll, afterAll } from "vitest"; +import { describe, expect, it, beforeAll, afterAll, vi } from "vitest"; import { execFile } from "child_process"; import { writeFile, unlink } from "fs/promises"; import { join, dirname } from "path"; @@ -159,6 +159,62 @@ describe("start-case-experiment-helper.cjs apparatus", () => { // ── D — Success output ────────────────────────────────────────── describe("D — success output", () => { + it("selects the existing OpenAI provider through the helper injection seam", async () => { + const { runStartCaseExperiment } = require(helperPath); + const previousProvider = process.env.START_CASE_EXPERIMENT_PROVIDER; + const previousKey = process.env.OPENAI_API_KEY; + process.env.START_CASE_EXPERIMENT_PROVIDER = "openai"; + process.env.OPENAI_API_KEY = "test-key"; + + try { + const result = await runStartCaseExperiment( + { scenario: "OpenAI provider selection scenario" }, + null, + { + startCase: async (_input, dependencies) => { + expect(typeof dependencies.reconstructionProvider?.generateReconstruction).toBe("function"); + expect(dependencies.reconstructionModelName).toBe("gpt-5.6-terra"); + return { + success: true, + situationGraph: { nodes: [], edges: [] }, + assessment: null, + selectedQuestion: null, + summary: "test", + }; + }, + }, + ); + + expect(result.success).toBe(true); + } finally { + if (previousProvider === undefined) delete process.env.START_CASE_EXPERIMENT_PROVIDER; + else process.env.START_CASE_EXPERIMENT_PROVIDER = previousProvider; + if (previousKey === undefined) delete process.env.OPENAI_API_KEY; + else process.env.OPENAI_API_KEY = previousKey; + } + }); + + it("fails before startCase when OpenAI is selected without a key", async () => { + const { runStartCaseExperiment } = require(helperPath); + const previousProvider = process.env.START_CASE_EXPERIMENT_PROVIDER; + const previousKey = process.env.OPENAI_API_KEY; + process.env.START_CASE_EXPERIMENT_PROVIDER = "openai"; + delete process.env.OPENAI_API_KEY; + const startCase = vi.fn(); + + try { + await expect( + runStartCaseExperiment({ scenario: "Missing key scenario" }, null, { startCase }), + ).rejects.toThrow("START_CASE_EXPERIMENT_PROVIDER=openai requires OPENAI_API_KEY"); + expect(startCase).not.toHaveBeenCalled(); + } finally { + if (previousProvider === undefined) delete process.env.START_CASE_EXPERIMENT_PROVIDER; + else process.env.START_CASE_EXPERIMENT_PROVIDER = previousProvider; + if (previousKey === undefined) delete process.env.OPENAI_API_KEY; + else process.env.OPENAI_API_KEY = previousKey; + } + }); + it("forwards an explicit reconstruction provider through the startCase path", async () => { const { runStartCaseExperiment } = require(helperPath); const reconstructionProvider = { generateReconstruction() {} };