diff --git a/docs/current-handoff.md b/docs/current-handoff.md index 4fe805a..67854ea 100644 --- a/docs/current-handoff.md +++ b/docs/current-handoff.md @@ -38,8 +38,8 @@ If YES, the next live experiment is one timed/costed OpenAI UI investigation mea - The OpenAI provider now honors a caller-supplied structured-output schema, applying its existing strict-schema transport projection; absent an alternate schema, initial reconstruction retains its existing strict schema and transport normalization. - The next live run exposed incomplete recursive strict projection: OpenAI rejected `relationships.items` because it lacked `additionalProperties: false`. The projector now recognizes every `type: "object"` node, including property-less objects in array items, and recursively enforces strict object schemas while preserving initial-reconstruction optionality/nullability behavior. - The latest Terra request then exposed an inconsistent root `properties`/`required` contract. The projector now derives `required` after projection from the surviving property keys, and recursive tests verify `properties`, `required`, and `additionalProperties` consistency. Property-less object strictness and initial-reconstruction transport behavior remain preserved. -- Provider and focused-route suites pass with zero live calls. -- Next boundary: retry one fresh real Playwright-driven OpenAI/Terra investigation; do not reuse the failed focused-deconstruction attempt. +- Current deterministic final-fetch schema remains internally valid, yet the live rejection contradicts it. `CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA=1` now emits one safe, server-side structural summary immediately before the OpenAI fetch—no prompt, answer, request body, secret, or model output. +- Provider suite passes with zero live calls. Next boundary: one fresh focused UI submission with the OpenAI provider and schema-trace flags enabled; capture the single trace and OpenAI response, with no Retry. ## Repository checkpoint diff --git a/lib/llm/provider.js b/lib/llm/provider.js index 2ac5dea..ef3987d 100644 --- a/lib/llm/provider.js +++ b/lib/llm/provider.js @@ -195,6 +195,55 @@ function projectOpenAIStrictSchema(schema, zodSchema, rootSchema, visited) { } } +function satisfiesOpenAIStrictSchema(schema, visited = new WeakSet()) { + if (!schema || typeof schema !== "object" || visited.has(schema)) return true; + visited.add(schema); + const isObject = schema.type === "object" || schema.properties; + if (isObject && schema.additionalProperties !== false) return false; + if (schema.properties) { + const propertyKeys = Object.keys(schema.properties); + if ( + !Array.isArray(schema.required) || + schema.required.length !== propertyKeys.length || + !propertyKeys.every((key) => schema.required.includes(key)) + ) return false; + } + return [ + schema.items, + ...Object.values(schema.properties ?? {}), + ...(schema.anyOf ?? []), + ...(schema.oneOf ?? []), + ...(schema.allOf ?? []), + ...Object.values(schema.$defs ?? schema.definitions ?? {}), + ].every((child) => satisfiesOpenAIStrictSchema(child, visited)); +} + +function traceOpenAISchema({ model, format, schema }) { + if (process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA !== "1") return; + const serializedSchema = JSON.stringify(schema); + const finalSchema = JSON.parse(serializedSchema); + const properties = finalSchema.properties ?? {}; + const required = finalSchema.required ?? []; + const relationships = properties.relationships; + console.info("[confidence-engine][openai-schema-trace]", { + model, + formatType: format.type, + formatName: format.name, + strict: format.strict, + rootProperties: Object.keys(properties), + rootRequired: required, + rootSetsEqual: + required.length === Object.keys(properties).length && + Object.keys(properties).every((key) => required.includes(key)), + relationshipsInProperties: Object.prototype.hasOwnProperty.call(properties, "relationships"), + relationshipsInRequired: required.includes("relationships"), + relationshipsItemType: relationships?.items?.type ?? null, + relationshipsItemAdditionalProperties: relationships?.items?.additionalProperties ?? null, + rootAdditionalProperties: finalSchema.additionalProperties ?? null, + recursiveStrictInvariant: satisfiesOpenAIStrictSchema(finalSchema), + }); +} + function schemaAllowsNull(schema, rootSchema) { const resolved = resolveSchema(schema, rootSchema); return ( @@ -479,6 +528,13 @@ class OpenAIReconstructionProvider { ? createOpenAIStrictSchema(outputSchema, null) : openAIReconstructionJsonSchema; + const format = { + type: "json_schema", + name: "reconstruction", + strict: true, + schema: structuredOutputSchema, + }; + traceOpenAISchema({ model: modelName, format, schema: structuredOutputSchema }); const response = await this.fetchImpl("https://api.openai.com/v1/responses", { method: "POST", headers: { @@ -489,12 +545,7 @@ class OpenAIReconstructionProvider { model: modelName, input: prompt, text: { - format: { - type: "json_schema", - name: "reconstruction", - strict: true, - schema: structuredOutputSchema, - }, + format, }, }), }); diff --git a/tests/llm/provider.test.js b/tests/llm/provider.test.js index 9d85d2d..1e925c2 100644 --- a/tests/llm/provider.test.js +++ b/tests/llm/provider.test.js @@ -194,6 +194,9 @@ describe("OpenAI reconstruction provider experiment seam", () => { if (!schema || typeof schema !== "object" || visited.has(schema)) return; visited.add(schema); + if (schema.type === "object" && !schema.properties) { + expect(schema.additionalProperties).toBe(false); + } if (schema.properties) { expect(schema.additionalProperties).toBe(false); expect(new Set(schema.required)).toEqual(new Set(Object.keys(schema.properties))); @@ -352,6 +355,8 @@ describe("OpenAI reconstruction provider experiment seam", () => { ); const request = JSON.parse(fetchSpy.mock.calls[0][1].body); const schema = request.text.format.schema; + const serializedSchema = JSON.stringify(schema); + const reparsedSchema = JSON.parse(serializedSchema); expect(schema.required).toEqual([ "targetNodeId", @@ -379,6 +384,69 @@ describe("OpenAI reconstruction provider experiment seam", () => { expect(schema.properties).toHaveProperty("relationships"); expect(schema.required).toContain("relationships"); assertStrictObjectInvariant(schema); + expect(Object.prototype.hasOwnProperty.call(schema.properties, "relationships")).toBe(true); + expect(reparsedSchema.required).toEqual(schema.required); + expect(Object.keys(reparsedSchema.properties)).toEqual(Object.keys(schema.properties)); + expect(Object.prototype.hasOwnProperty.call(reparsedSchema.properties, "relationships")).toBe(true); + expect(reparsedSchema.properties.relationships).toEqual(schema.properties.relationships); + expect(reparsedSchema.additionalProperties).toBe(false); + assertStrictObjectInvariant(reparsedSchema); + }); + + it("emits one safe final-schema trace only when explicitly enabled", async () => { + const previousTraceFlag = process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA; + const infoSpy = vi.spyOn(console, "info").mockImplementation(() => {}); + const fetchSpy = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ output_text: JSON.stringify({ + targetNodeId: "target", observations: [], uncertainties: [], assumptions: [], relationships: [], possibleFollowUpQuestions: [], + }) }), + }); + process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA = "1"; + + try { + const provider = createOpenAIReconstructionProvider({ apiKey: "test-key", fetchImpl: fetchSpy }); + await provider.generateReconstruction("focused prompt", "gpt-5.6-terra", focusedDeconstructJsonSchema); + expect(infoSpy).toHaveBeenCalledTimes(1); + const [label, trace] = infoSpy.mock.calls[0]; + expect(label).toBe("[confidence-engine][openai-schema-trace]"); + expect(trace).toMatchObject({ + model: "gpt-5.6-terra", + formatType: "json_schema", + formatName: "reconstruction", + strict: true, + rootSetsEqual: true, + relationshipsInProperties: true, + relationshipsInRequired: true, + relationshipsItemAdditionalProperties: false, + rootAdditionalProperties: false, + recursiveStrictInvariant: true, + }); + expect(JSON.stringify(trace)).not.toContain("focused prompt"); + } finally { + infoSpy.mockRestore(); + if (previousTraceFlag === undefined) delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA; + else process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA = previousTraceFlag; + } + }); + + it("does not emit a schema trace when the flag is absent", async () => { + const previousTraceFlag = process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA; + const infoSpy = vi.spyOn(console, "info").mockImplementation(() => {}); + delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA; + const provider = createOpenAIReconstructionProvider({ + apiKey: "test-key", + fetchImpl: vi.fn().mockResolvedValue({ ok: true, json: async () => ({ output_text: "{}" }) }), + }); + + try { + await provider.generateReconstruction("prompt"); + expect(infoSpy).not.toHaveBeenCalled(); + } finally { + infoSpy.mockRestore(); + if (previousTraceFlag === undefined) delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA; + else process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA = previousTraceFlag; + } }); it("recursively projects arbitrary nested object schemas as strict", () => {