experiment(confidence-engine): trace live openai schema boundary
This commit is contained in:
@@ -194,6 +194,9 @@ describe("OpenAI reconstruction provider experiment seam", () => {
|
||||
if (!schema || typeof schema !== "object" || visited.has(schema)) return;
|
||||
visited.add(schema);
|
||||
|
||||
if (schema.type === "object" && !schema.properties) {
|
||||
expect(schema.additionalProperties).toBe(false);
|
||||
}
|
||||
if (schema.properties) {
|
||||
expect(schema.additionalProperties).toBe(false);
|
||||
expect(new Set(schema.required)).toEqual(new Set(Object.keys(schema.properties)));
|
||||
@@ -352,6 +355,8 @@ describe("OpenAI reconstruction provider experiment seam", () => {
|
||||
);
|
||||
const request = JSON.parse(fetchSpy.mock.calls[0][1].body);
|
||||
const schema = request.text.format.schema;
|
||||
const serializedSchema = JSON.stringify(schema);
|
||||
const reparsedSchema = JSON.parse(serializedSchema);
|
||||
|
||||
expect(schema.required).toEqual([
|
||||
"targetNodeId",
|
||||
@@ -379,6 +384,69 @@ describe("OpenAI reconstruction provider experiment seam", () => {
|
||||
expect(schema.properties).toHaveProperty("relationships");
|
||||
expect(schema.required).toContain("relationships");
|
||||
assertStrictObjectInvariant(schema);
|
||||
expect(Object.prototype.hasOwnProperty.call(schema.properties, "relationships")).toBe(true);
|
||||
expect(reparsedSchema.required).toEqual(schema.required);
|
||||
expect(Object.keys(reparsedSchema.properties)).toEqual(Object.keys(schema.properties));
|
||||
expect(Object.prototype.hasOwnProperty.call(reparsedSchema.properties, "relationships")).toBe(true);
|
||||
expect(reparsedSchema.properties.relationships).toEqual(schema.properties.relationships);
|
||||
expect(reparsedSchema.additionalProperties).toBe(false);
|
||||
assertStrictObjectInvariant(reparsedSchema);
|
||||
});
|
||||
|
||||
it("emits one safe final-schema trace only when explicitly enabled", async () => {
|
||||
const previousTraceFlag = process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA;
|
||||
const infoSpy = vi.spyOn(console, "info").mockImplementation(() => {});
|
||||
const fetchSpy = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ output_text: JSON.stringify({
|
||||
targetNodeId: "target", observations: [], uncertainties: [], assumptions: [], relationships: [], possibleFollowUpQuestions: [],
|
||||
}) }),
|
||||
});
|
||||
process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA = "1";
|
||||
|
||||
try {
|
||||
const provider = createOpenAIReconstructionProvider({ apiKey: "test-key", fetchImpl: fetchSpy });
|
||||
await provider.generateReconstruction("focused prompt", "gpt-5.6-terra", focusedDeconstructJsonSchema);
|
||||
expect(infoSpy).toHaveBeenCalledTimes(1);
|
||||
const [label, trace] = infoSpy.mock.calls[0];
|
||||
expect(label).toBe("[confidence-engine][openai-schema-trace]");
|
||||
expect(trace).toMatchObject({
|
||||
model: "gpt-5.6-terra",
|
||||
formatType: "json_schema",
|
||||
formatName: "reconstruction",
|
||||
strict: true,
|
||||
rootSetsEqual: true,
|
||||
relationshipsInProperties: true,
|
||||
relationshipsInRequired: true,
|
||||
relationshipsItemAdditionalProperties: false,
|
||||
rootAdditionalProperties: false,
|
||||
recursiveStrictInvariant: true,
|
||||
});
|
||||
expect(JSON.stringify(trace)).not.toContain("focused prompt");
|
||||
} finally {
|
||||
infoSpy.mockRestore();
|
||||
if (previousTraceFlag === undefined) delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA;
|
||||
else process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA = previousTraceFlag;
|
||||
}
|
||||
});
|
||||
|
||||
it("does not emit a schema trace when the flag is absent", async () => {
|
||||
const previousTraceFlag = process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA;
|
||||
const infoSpy = vi.spyOn(console, "info").mockImplementation(() => {});
|
||||
delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA;
|
||||
const provider = createOpenAIReconstructionProvider({
|
||||
apiKey: "test-key",
|
||||
fetchImpl: vi.fn().mockResolvedValue({ ok: true, json: async () => ({ output_text: "{}" }) }),
|
||||
});
|
||||
|
||||
try {
|
||||
await provider.generateReconstruction("prompt");
|
||||
expect(infoSpy).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
infoSpy.mockRestore();
|
||||
if (previousTraceFlag === undefined) delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA;
|
||||
else process.env.CONFIDENCE_ENGINE_EXPERIMENT_TRACE_OPENAI_SCHEMA = previousTraceFlag;
|
||||
}
|
||||
});
|
||||
|
||||
it("recursively projects arbitrary nested object schemas as strict", () => {
|
||||
|
||||
Reference in New Issue
Block a user