diff --git a/app/api/cases/start/route.js b/app/api/cases/start/route.js index 563fc22..69913ba 100644 --- a/app/api/cases/start/route.js +++ b/app/api/cases/start/route.js @@ -23,6 +23,7 @@ export async function POST(request) { analysisErrors: result.analysisErrors, validationIssues: result.validationIssues, providerApiPath: result.providerApiPath, + providerExecution: result.providerExecution, rawResponse: result.rawResponse ?? undefined, }; @@ -41,6 +42,7 @@ export async function POST(request) { analysisErrors: result.analysisErrors, validationIssues: result.validationIssues, providerApiPath: result.providerApiPath, + providerExecution: result.providerExecution, rawResponse: result.rawResponse ?? undefined, }, { status }, diff --git a/docs/current-handoff.md b/docs/current-handoff.md index d6a6973..2e7479b 100644 --- a/docs/current-handoff.md +++ b/docs/current-handoff.md @@ -86,6 +86,12 @@ - The deterministic provider-boundary test passes 2/2 under Node. A direct configured Ollama/Qwen `/api/chat` call accepted and obeyed the full schema in one call; `/api/generate`, chat detection, fallback, prompt, temperature, retries, and schema semantics are unchanged. - Production repeatability remains untested after this change. Next restart point: a small repeated `/api/cases/start` stability observation using the fixed manufacturing scenario. +## Reconstruction fallback diagnostics + +- Schema-constrained reconstruction `/api/chat` remains active. A later manual production 502 came through `/api/generate`; its unconstrained response omitted required transition confidence fields and failed Zod validation. +- Provider execution diagnostics now distinguish chat skipped due capability state, chat attempted and failed before generate fallback, and successful chat without fallback. They are deterministically verified through `/api/cases/start`; endpoint selection and fallback behaviour remain unchanged. +- Next restart point: one observation-only fixed-scenario production call to identify why `/api/generate` is reached. + ## Current product architecture Three distinct routes, not a single page: diff --git a/lib/analysis.js b/lib/analysis.js index 05401a5..f75579f 100644 --- a/lib/analysis.js +++ b/lib/analysis.js @@ -73,6 +73,7 @@ export async function analyseScenario(scenario, opts = {}) { const provider = getProvider(); let rawResponse; let providerApiPath; + let providerExecution; try { const providerResult = await provider.generateReconstruction( promptObj.prompt, @@ -86,6 +87,7 @@ export async function analyseScenario(scenario, opts = {}) { ) { rawResponse = providerResult.response; providerApiPath = providerResult.providerApiPath; + providerExecution = providerResult.providerExecution; } else { rawResponse = providerResult; } @@ -95,6 +97,7 @@ export async function analyseScenario(scenario, opts = {}) { Date.now() - startTime, "500", e.providerApiPath, + e.providerExecution, ); } @@ -150,6 +153,7 @@ export async function analyseScenario(scenario, opts = {}) { promptVersion, compatibility, providerApiPath, + providerExecution, ); } @@ -169,7 +173,7 @@ function tryValidateAgainstSchema(data, schema) { // ── Result builders ────────────────────────────────── -function buildErrorResponse(message, elapsed, statusCode = 500, providerApiPath) { +function buildErrorResponse(message, elapsed, statusCode = 500, providerApiPath, providerExecution) { return { success: false, error: message, @@ -180,6 +184,7 @@ function buildErrorResponse(message, elapsed, statusCode = 500, providerApiPath) promptVersion: null, statusCode, providerApiPath, + providerExecution, }; } @@ -233,6 +238,7 @@ function buildPartialResult( version, compatibility, providerApiPath, + providerExecution, ) { let errors = []; const validationIssues = error?.issues ?? []; @@ -260,6 +266,7 @@ function buildPartialResult( errors, validationIssues, providerApiPath, + providerExecution, ...buildCompatibilityDiagnostics(compatibility), }; } diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index 09a877b..966ff47 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -373,6 +373,7 @@ export async function startCase(body, dependencies = {}) { analysisErrors: analysis.errors ?? undefined, validationIssues: analysis.validationIssues ?? undefined, providerApiPath: analysis.providerApiPath ?? undefined, + providerExecution: analysis.providerExecution ?? undefined, rawResponse: analysis.rawResponse ?? undefined, statusCode: Number(analysis.statusCode) || 502, }; diff --git a/lib/llm/provider.js b/lib/llm/provider.js index 1cb6737..fcca1f3 100644 --- a/lib/llm/provider.js +++ b/lib/llm/provider.js @@ -64,6 +64,11 @@ function recoverJson(raw) { let _chatSupported = null; const reconstructionJsonSchema = z.toJSONSchema(reconstructionV2Schema); +/** @internal Test-only seam for isolated provider capability scenarios. */ +export function __resetChatSupportForTests() { + _chatSupported = null; +} + async function detectChatSupport(baseUrl, modelName) { if (_chatSupported !== null) return _chatSupported; @@ -108,6 +113,12 @@ class OllamaLlmProvider { let chatSupported = false; let rawResponse = null; let fullResponseData = null; + const providerExecution = { + chatCapabilityDetected: false, + chatRequestAttempted: false, + chatRequestSucceeded: false, + generateRequestAttempted: false, + }; // ================================================================ // Step 1: Detect whether /api/chat exists (cache result) @@ -115,6 +126,7 @@ class OllamaLlmProvider { try { chatSupported = await detectChatSupport(baseUrl, modelName); } catch { /* failed silently — defaults to false */ } + providerExecution.chatCapabilityDetected = chatSupported; // ================================================================ // Step 2: Try /api/chat if supported with the reconstruction schema @@ -125,6 +137,7 @@ class OllamaLlmProvider { const timeout = setTimeout(() => controller.abort(), 60000); apiUsed = "/api/chat"; + providerExecution.chatRequestAttempted = true; const res = await fetch(`${baseUrl}/api/chat`, { method: "POST", headers: { "Content-Type": "application/json" }, @@ -140,6 +153,7 @@ class OllamaLlmProvider { clearTimeout(timeout); if (res.ok) { + providerExecution.chatRequestSucceeded = true; fullResponseData = await res.json(); rawResponse = typeof fullResponseData.message?.content === "string" ? fullResponseData.message.content @@ -163,6 +177,7 @@ class OllamaLlmProvider { const timeout = setTimeout(() => controller.abort(), 300000); // 5 min for cold start apiUsed = "/api/generate"; // set BEFORE the request so we know which API failed + providerExecution.generateRequestAttempted = true; const res = await fetch(`${baseUrl}/api/generate`, { method: "POST", @@ -204,6 +219,7 @@ class OllamaLlmProvider { `- Check Ollama logs: \`ollama serve\` or look at your system logs` ); error.providerApiPath = apiUsed; + error.providerExecution = providerExecution; throw error; } throw e; @@ -244,6 +260,7 @@ class OllamaLlmProvider { "- If your model is very small (e.g., tinyllama, phi), try a larger one like llama3.1 or mistral" ); error.providerApiPath = apiUsed; + error.providerExecution = providerExecution; throw error; } @@ -254,6 +271,7 @@ class OllamaLlmProvider { return { response: recoverJson(rawResponse), providerApiPath: apiUsed, + providerExecution, }; } catch (e) { if (e instanceof SyntaxError) { @@ -269,6 +287,7 @@ class OllamaLlmProvider { "- Consider upgrading Ollama: https://ollama.com/download" ); error.providerApiPath = apiUsed; + error.providerExecution = providerExecution; throw error; } throw e; diff --git a/tests/app/api/cases-start-route.test.js b/tests/app/api/cases-start-route.test.js index 84930a0..3dca41a 100644 --- a/tests/app/api/cases-start-route.test.js +++ b/tests/app/api/cases-start-route.test.js @@ -139,6 +139,12 @@ describe("app/api/cases/start route", () => { }, ], providerApiPath: "/api/generate", + providerExecution: { + chatCapabilityDetected: false, + chatRequestAttempted: false, + chatRequestSucceeded: false, + generateRequestAttempted: true, + }, rawResponse, }); @@ -167,6 +173,12 @@ describe("app/api/cases/start route", () => { }), ]); expect(body.providerApiPath).toBe("/api/generate"); + expect(body.providerExecution).toEqual({ + chatCapabilityDetected: false, + chatRequestAttempted: false, + chatRequestSucceeded: false, + generateRequestAttempted: true, + }); expect(errorSpy).toHaveBeenCalledTimes(1); expect(errorSpy).toHaveBeenCalledWith( "[api/cases/start] error response", @@ -176,6 +188,12 @@ describe("app/api/cases/start route", () => { analysisErrors: ["reconstruction: Required"], validationIssues: expect.any(Array), providerApiPath: "/api/generate", + providerExecution: { + chatCapabilityDetected: false, + chatRequestAttempted: false, + chatRequestSucceeded: false, + generateRequestAttempted: true, + }, rawResponse, }), ); diff --git a/tests/graph/orchestrator.test.js b/tests/graph/orchestrator.test.js index 5fb499a..96b4610 100644 --- a/tests/graph/orchestrator.test.js +++ b/tests/graph/orchestrator.test.js @@ -567,10 +567,17 @@ describe("lib/graph/orchestrator startCase", () => { }); it("preserves an attempted provider API path on analysis failure", async () => { + const providerExecution = { + chatCapabilityDetected: true, + chatRequestAttempted: true, + chatRequestSucceeded: false, + generateRequestAttempted: true, + }; mockAnalyseScenario.mockResolvedValue({ success: false, error: "Provider failed", providerApiPath: "/api/chat", + providerExecution, }); const { startCase } = await import("@/lib/graph/orchestrator.js"); @@ -580,6 +587,7 @@ describe("lib/graph/orchestrator startCase", () => { success: false, statusCode: 502, providerApiPath: "/api/chat", + providerExecution, }); }); diff --git a/tests/llm/provider.test.js b/tests/llm/provider.test.js index 7ed6d01..e23a198 100644 --- a/tests/llm/provider.test.js +++ b/tests/llm/provider.test.js @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from "vitest"; +import { __resetChatSupportForTests, getProvider } from "@/lib/llm/provider.js"; describe("OllamaLlmProvider chat capability detection", () => { it("uses the configured model for the chat probe and keeps the chat path", async () => { @@ -13,7 +14,7 @@ describe("OllamaLlmProvider chat capability detection", () => { process.env.OLLAMA_BASE_URL = "http://ollama.test"; try { - const { getProvider } = await import("@/lib/llm/provider.js"); + __resetChatSupportForTests(); const result = await getProvider().generateReconstruction("prompt", "configured-model"); expect(JSON.parse(fetchSpy.mock.calls[0][1].body)).toMatchObject({ @@ -37,6 +38,12 @@ describe("OllamaLlmProvider chat capability detection", () => { expect(result).toMatchObject({ response: {}, providerApiPath: "/api/chat", + providerExecution: { + chatCapabilityDetected: true, + chatRequestAttempted: true, + chatRequestSucceeded: true, + generateRequestAttempted: false, + }, }); } finally { vi.unstubAllGlobals(); @@ -45,7 +52,7 @@ describe("OllamaLlmProvider chat capability detection", () => { } }); - it("reports the actually attempted generate path when generation fails", async () => { + it("reports chat-skipped generate fallback execution", async () => { const originalBaseUrl = process.env.OLLAMA_BASE_URL; const fetchSpy = vi.fn() .mockResolvedValueOnce({ ok: false, status: 501, body: { consume: vi.fn() } }) @@ -54,10 +61,18 @@ describe("OllamaLlmProvider chat capability detection", () => { process.env.OLLAMA_BASE_URL = "http://ollama.test"; try { - const { getProvider } = await import("@/lib/llm/provider.js"); + __resetChatSupportForTests(); await expect( getProvider().generateReconstruction("prompt", "configured-model"), - ).rejects.toMatchObject({ providerApiPath: "/api/generate" }); + ).rejects.toMatchObject({ + providerApiPath: "/api/generate", + providerExecution: { + chatCapabilityDetected: false, + chatRequestAttempted: false, + chatRequestSucceeded: false, + generateRequestAttempted: true, + }, + }); expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/generate"); } finally { vi.unstubAllGlobals(); @@ -65,4 +80,35 @@ describe("OllamaLlmProvider chat capability detection", () => { else process.env.OLLAMA_BASE_URL = originalBaseUrl; } }); + + it("reports chat-attempt-failed generate fallback execution", async () => { + const originalBaseUrl = process.env.OLLAMA_BASE_URL; + const fetchSpy = vi.fn() + .mockResolvedValueOnce({ ok: true, body: { consume: vi.fn() } }) + .mockResolvedValueOnce({ ok: false, body: { consume: vi.fn() } }) + .mockResolvedValueOnce({ ok: false, status: 500, text: async () => "failure" }); + vi.stubGlobal("fetch", fetchSpy); + process.env.OLLAMA_BASE_URL = "http://ollama.test"; + + try { + __resetChatSupportForTests(); + await expect( + getProvider().generateReconstruction("prompt", "configured-model"), + ).rejects.toMatchObject({ + providerApiPath: "/api/generate", + providerExecution: { + chatCapabilityDetected: true, + chatRequestAttempted: true, + chatRequestSucceeded: false, + generateRequestAttempted: true, + }, + }); + expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/chat"); + expect(fetchSpy.mock.calls[2][0]).toBe("http://ollama.test/api/generate"); + } finally { + vi.unstubAllGlobals(); + if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL; + else process.env.OLLAMA_BASE_URL = originalBaseUrl; + } + }); }); \ No newline at end of file diff --git a/tests/reconstruction/compatibility.test.js b/tests/reconstruction/compatibility.test.js index e7454e8..8ec78c8 100644 --- a/tests/reconstruction/compatibility.test.js +++ b/tests/reconstruction/compatibility.test.js @@ -115,6 +115,12 @@ describe("analyseScenario compatibility", () => { it("preserves an attempted provider API path on provider failure", async () => { const providerError = new Error("Provider failed"); providerError.providerApiPath = "/api/chat"; + providerError.providerExecution = { + chatCapabilityDetected: true, + chatRequestAttempted: true, + chatRequestSucceeded: false, + generateRequestAttempted: true, + }; mockGenerateReconstruction.mockRejectedValue(providerError); const { analyseScenario } = await import("@/lib/analysis.js"); @@ -125,6 +131,7 @@ describe("analyseScenario compatibility", () => { expect(result).toMatchObject({ success: false, providerApiPath: "/api/chat", + providerExecution: providerError.providerExecution, }); });