experiment(confidence-engine): trace live openai schema boundary

This commit is contained in:
2026-09-07 12:51:13 +01:00
parent 0d27d4935b
commit 0cbe49913e
3 changed files with 127 additions and 8 deletions
+68
View File
@@ -194,6 +194,9 @@ describe("OpenAI reconstruction provider experiment seam", () => {
if (!schema || typeof schema !== "object" || visited.has(schema)) return;
visited.add(schema);
if (schema.type === "object" && !schema.properties) {
expect(schema.additionalProperties).toBe(false);
}
if (schema.properties) {
expect(schema.additionalProperties).toBe(false);
expect(new Set(schema.required)).toEqual(new Set(Object.keys(schema.properties)));
@@ -352,6 +355,8 @@ describe("OpenAI reconstruction provider experiment seam", () => {
);
const request = JSON.parse(fetchSpy.mock.calls[0][1].body);
const schema = request.text.format.schema;
const serializedSchema = JSON.stringify(schema);
const reparsedSchema = JSON.parse(serializedSchema);
expect(schema.required).toEqual([
"targetNodeId",
@@ -379,6 +384,69 @@ describe("OpenAI reconstruction provider experiment seam", () => {
expect(schema.properties).toHaveProperty("relationships");
expect(schema.required).toContain("relationships");
assertStrictObjectInvariant(schema);
expect(Object.prototype.hasOwnProperty.call(schema.properties, "relationships")).toBe(true);
expect(reparsedSchema.required).toEqual(schema.required);
expect(Object.keys(reparsedSchema.properties)).toEqual(Object.keys(schema.properties));
expect(Object.prototype.hasOwnProperty.call(reparsedSchema.properties, "relationships")).toBe(true);
expect(reparsedSchema.properties.relationships).toEqual(schema.properties.relationships);
expect(reparsedSchema.additionalProperties).toBe(false);
assertStrictObjectInvariant(reparsedSchema);
});
it("emits one safe final-schema trace only when explicitly enabled", async () => {
const previousTraceFlag = process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA;
const infoSpy = vi.spyOn(console, "info").mockImplementation(() => {});
const fetchSpy = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ output_text: JSON.stringify({
targetNodeId: "target", observations: [], uncertainties: [], assumptions: [], relationships: [], possibleFollowUpQuestions: [],
}) }),
});
process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA = "1";
try {
const provider = createOpenAIReconstructionProvider({ apiKey: "test-key", fetchImpl: fetchSpy });
await provider.generateReconstruction("focused prompt", "gpt-5.6-terra", focusedDeconstructJsonSchema);
expect(infoSpy).toHaveBeenCalledTimes(1);
const [label, trace] = infoSpy.mock.calls[0];
expect(label).toBe("[confidence-engine][openai-schema-trace]");
expect(trace).toMatchObject({
model: "gpt-5.6-terra",
formatType: "json_schema",
formatName: "reconstruction",
strict: true,
rootSetsEqual: true,
relationshipsInProperties: true,
relationshipsInRequired: true,
relationshipsItemAdditionalProperties: false,
rootAdditionalProperties: false,
recursiveStrictInvariant: true,
});
expect(JSON.stringify(trace)).not.toContain("focused prompt");
} finally {
infoSpy.mockRestore();
if (previousTraceFlag === undefined) delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA;
else process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA = previousTraceFlag;
}
});
it("does not emit a schema trace when the flag is absent", async () => {
const previousTraceFlag = process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA;
const infoSpy = vi.spyOn(console, "info").mockImplementation(() => {});
delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA;
const provider = createOpenAIReconstructionProvider({
apiKey: "test-key",
fetchImpl: vi.fn().mockResolvedValue({ ok: true, json: async () => ({ output_text: "{}" }) }),
});
try {
await provider.generateReconstruction("prompt");
expect(infoSpy).not.toHaveBeenCalled();
} finally {
infoSpy.mockRestore();
if (previousTraceFlag === undefined) delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA;
else process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA = previousTraceFlag;
}
});
it("recursively projects arbitrary nested object schemas as strict", () => {