From c7a0a79d0f56d0c482430cbc7fb1eb566b2812ce Mon Sep 17 00:00:00 2001 From: robbond Date: Sat, 5 Sep 2026 18:44:47 +0100 Subject: [PATCH] feat(confidence-engine): constrain reconstruction chat output --- docs/current-handoff.md | 4 +++- lib/llm/provider.js | 8 ++++++-- tests/llm/provider.test.js | 11 +++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/current-handoff.md b/docs/current-handoff.md index c10adeb..d6a6973 100644 --- a/docs/current-handoff.md +++ b/docs/current-handoff.md @@ -82,7 +82,9 @@ - Zod upgraded from 3.25.76 to 4.5.4. Reconstruction and direct Zod-owner suites pass, and native `z.toJSONSchema()` converts the production reconstruction schema with required closed enum constraints. - Zod-3-specific error-message assertions are version-independent while retaining semantic validation evidence. Production schema semantics are unchanged and no third-party JSON-schema conversion dependency was added. -- Next unknown: whether Ollama/Qwen accepts and obeys the generated schema through `/api/chat` format. +- Reconstruction `/api/chat` now sends `z.toJSONSchema(reconstructionV2Schema)` rather than generic `format: "json"`; the same canonical Zod contract constrains generation and remains the post-hoc validator. +- The deterministic provider-boundary test passes 2/2 under Node. A direct configured Ollama/Qwen `/api/chat` call accepted and obeyed the full schema in one call; `/api/generate`, chat detection, fallback, prompt, temperature, retries, and schema semantics are unchanged. +- Production repeatability remains untested after this change. Next restart point: a small repeated `/api/cases/start` stability observation using the fixed manufacturing scenario. ## Current product architecture diff --git a/lib/llm/provider.js b/lib/llm/provider.js index 4d1057b..1cb6737 100644 --- a/lib/llm/provider.js +++ b/lib/llm/provider.js @@ -1,3 +1,6 @@ +import { z } from "zod"; +import { reconstructionV2Schema } from "../reconstruction/schema.js"; + /** * Provider abstraction — the app calls getProvider() which returns an object * with a generateReconstruction(scenario, modelName) method. @@ -59,6 +62,7 @@ function recoverJson(raw) { } let _chatSupported = null; +const reconstructionJsonSchema = z.toJSONSchema(reconstructionV2Schema); async function detectChatSupport(baseUrl, modelName) { if (_chatSupported !== null) return _chatSupported; @@ -113,7 +117,7 @@ class OllamaLlmProvider { } catch { /* failed silently — defaults to false */ } // ================================================================ - // Step 2: Try /api/chat if supported and format:json works + // Step 2: Try /api/chat if supported with the reconstruction schema // ================================================================ if (chatSupported) { try { @@ -128,7 +132,7 @@ class OllamaLlmProvider { model: modelName, messages: [{ role: "user", content: prompt }], stream: false, - format: "json", + format: reconstructionJsonSchema, }), signal: controller.signal, }); diff --git a/tests/llm/provider.test.js b/tests/llm/provider.test.js index 60056c6..7ed6d01 100644 --- a/tests/llm/provider.test.js +++ b/tests/llm/provider.test.js @@ -23,6 +23,17 @@ describe("OllamaLlmProvider chat capability detection", () => { expect(fetchSpy.mock.calls[0][0]).toBe("http://ollama.test/api/chat"); expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/chat"); expect(fetchSpy.mock.calls[1][0]).not.toContain("/api/generate"); + const chatRequest = JSON.parse(fetchSpy.mock.calls[1][1].body); + expect(chatRequest).toMatchObject({ + model: "configured-model", + messages: [{ role: "user", content: "prompt" }], + stream: false, + }); + expect(chatRequest.format).toBeTypeOf("object"); + expect(chatRequest.format).not.toBe("json"); + const formatText = JSON.stringify(chatRequest.format); + expect(formatText).toContain("relationship"); + expect(formatText).toContain("evidenceType"); expect(result).toMatchObject({ response: {}, providerApiPath: "/api/chat",