chore(confidence-engine): expose reconstruction fallback path

app/api/cases/start/route.js

@@ -23,6 +23,7 @@ export async function POST(request) {
       analysisErrors: result.analysisErrors,
       validationIssues: result.validationIssues,
       providerApiPath: result.providerApiPath,
+      providerExecution: result.providerExecution,
       rawResponse: result.rawResponse ?? undefined,
     };
 
@@ -41,6 +42,7 @@ export async function POST(request) {
         analysisErrors: result.analysisErrors,
         validationIssues: result.validationIssues,
         providerApiPath: result.providerApiPath,
+        providerExecution: result.providerExecution,
         rawResponse: result.rawResponse ?? undefined,
       },
       { status },

docs/current-handoff.md

@@ -86,6 +86,12 @@
 - The deterministic provider-boundary test passes 2/2 under Node. A direct configured Ollama/Qwen `/api/chat` call accepted and obeyed the full schema in one call; `/api/generate`, chat detection, fallback, prompt, temperature, retries, and schema semantics are unchanged.
 - Production repeatability remains untested after this change. Next restart point: a small repeated `/api/cases/start` stability observation using the fixed manufacturing scenario.
 
+## Reconstruction fallback diagnostics
+
+- Schema-constrained reconstruction `/api/chat` remains active. A later manual production 502 came through `/api/generate`; its unconstrained response omitted required transition confidence fields and failed Zod validation.
+- Provider execution diagnostics now distinguish chat skipped due capability state, chat attempted and failed before generate fallback, and successful chat without fallback. They are deterministically verified through `/api/cases/start`; endpoint selection and fallback behaviour remain unchanged.
+- Next restart point: one observation-only fixed-scenario production call to identify why `/api/generate` is reached.
+
 ## Current product architecture
 
 Three distinct routes, not a single page:

lib/analysis.js

@@ -73,6 +73,7 @@ export async function analyseScenario(scenario, opts = {}) {
   const provider = getProvider();
   let rawResponse;
   let providerApiPath;
+  let providerExecution;
   try {
     const providerResult = await provider.generateReconstruction(
       promptObj.prompt,
@@ -86,6 +87,7 @@ export async function analyseScenario(scenario, opts = {}) {
     ) {
       rawResponse = providerResult.response;
       providerApiPath = providerResult.providerApiPath;
+      providerExecution = providerResult.providerExecution;
     } else {
       rawResponse = providerResult;
     }
@@ -95,6 +97,7 @@ export async function analyseScenario(scenario, opts = {}) {
       Date.now() - startTime,
       "500",
       e.providerApiPath,
+      e.providerExecution,
     );
   }
 
@@ -150,6 +153,7 @@ export async function analyseScenario(scenario, opts = {}) {
     promptVersion,
     compatibility,
     providerApiPath,
+    providerExecution,
   );
 }
 
@@ -169,7 +173,7 @@ function tryValidateAgainstSchema(data, schema) {
 
 // ── Result builders ──────────────────────────────────
 
-function buildErrorResponse(message, elapsed, statusCode = 500, providerApiPath) {
+function buildErrorResponse(message, elapsed, statusCode = 500, providerApiPath, providerExecution) {
   return {
     success: false,
     error: message,
@@ -180,6 +184,7 @@ function buildErrorResponse(message, elapsed, statusCode = 500, providerApiPath)
     promptVersion: null,
     statusCode,
     providerApiPath,
+    providerExecution,
   };
 }
 
@@ -233,6 +238,7 @@ function buildPartialResult(
   version,
   compatibility,
   providerApiPath,
+  providerExecution,
 ) {
   let errors = [];
   const validationIssues = error?.issues ?? [];
@@ -260,6 +266,7 @@ function buildPartialResult(
     errors,
     validationIssues,
     providerApiPath,
+    providerExecution,
     ...buildCompatibilityDiagnostics(compatibility),
   };
 }

lib/graph/orchestrator.js

@@ -373,6 +373,7 @@ export async function startCase(body, dependencies = {}) {
       analysisErrors: analysis.errors ?? undefined,
       validationIssues: analysis.validationIssues ?? undefined,
       providerApiPath: analysis.providerApiPath ?? undefined,
+      providerExecution: analysis.providerExecution ?? undefined,
       rawResponse: analysis.rawResponse ?? undefined,
       statusCode: Number(analysis.statusCode) || 502,
     };

lib/llm/provider.js

@@ -64,6 +64,11 @@ function recoverJson(raw) {
 let _chatSupported = null;
 const reconstructionJsonSchema = z.toJSONSchema(reconstructionV2Schema);
 
+/** @internal Test-only seam for isolated provider capability scenarios. */
+export function __resetChatSupportForTests() {
+  _chatSupported = null;
+}
+
 async function detectChatSupport(baseUrl, modelName) {
   if (_chatSupported !== null) return _chatSupported;
   
@@ -108,6 +113,12 @@ class OllamaLlmProvider {
     let chatSupported = false;
     let rawResponse = null;
     let fullResponseData = null;
+    const providerExecution = {
+      chatCapabilityDetected: false,
+      chatRequestAttempted: false,
+      chatRequestSucceeded: false,
+      generateRequestAttempted: false,
+    };
 
     // ================================================================
     // Step 1: Detect whether /api/chat exists (cache result)
@@ -115,6 +126,7 @@ class OllamaLlmProvider {
     try {
       chatSupported = await detectChatSupport(baseUrl, modelName);
     } catch { /* failed silently — defaults to false */ }
+    providerExecution.chatCapabilityDetected = chatSupported;
 
     // ================================================================
     // Step 2: Try /api/chat if supported with the reconstruction schema
@@ -125,6 +137,7 @@ class OllamaLlmProvider {
         const timeout = setTimeout(() => controller.abort(), 60000);
 
         apiUsed = "/api/chat";
+        providerExecution.chatRequestAttempted = true;
         const res = await fetch(`${baseUrl}/api/chat`, {
           method: "POST",
           headers: { "Content-Type": "application/json" },
@@ -140,6 +153,7 @@ class OllamaLlmProvider {
         clearTimeout(timeout);
 
         if (res.ok) {
+          providerExecution.chatRequestSucceeded = true;
           fullResponseData = await res.json();
           rawResponse = typeof fullResponseData.message?.content === "string"
             ? fullResponseData.message.content
@@ -163,6 +177,7 @@ class OllamaLlmProvider {
         const timeout = setTimeout(() => controller.abort(), 300000); // 5 min for cold start
 
         apiUsed = "/api/generate"; // set BEFORE the request so we know which API failed
+        providerExecution.generateRequestAttempted = true;
 
         const res = await fetch(`${baseUrl}/api/generate`, {
           method: "POST",
@@ -204,6 +219,7 @@ class OllamaLlmProvider {
             `- Check Ollama logs: \`ollama serve\` or look at your system logs`
           );
           error.providerApiPath = apiUsed;
+          error.providerExecution = providerExecution;
           throw error;
         }
         throw e;
@@ -244,6 +260,7 @@ class OllamaLlmProvider {
         "- If your model is very small (e.g., tinyllama, phi), try a larger one like llama3.1 or mistral"
       );
       error.providerApiPath = apiUsed;
+      error.providerExecution = providerExecution;
       throw error;
     }
 
@@ -254,6 +271,7 @@ class OllamaLlmProvider {
       return {
         response: recoverJson(rawResponse),
         providerApiPath: apiUsed,
+        providerExecution,
       };
     } catch (e) {
       if (e instanceof SyntaxError) {
@@ -269,6 +287,7 @@ class OllamaLlmProvider {
           "- Consider upgrading Ollama: https://ollama.com/download"
         );
         error.providerApiPath = apiUsed;
+        error.providerExecution = providerExecution;
         throw error;
       }
       throw e;

tests/app/api/cases-start-route.test.js

@@ -139,6 +139,12 @@ describe("app/api/cases/start route", () => {
         },
       ],
       providerApiPath: "/api/generate",
+      providerExecution: {
+        chatCapabilityDetected: false,
+        chatRequestAttempted: false,
+        chatRequestSucceeded: false,
+        generateRequestAttempted: true,
+      },
       rawResponse,
     });
 
@@ -167,6 +173,12 @@ describe("app/api/cases/start route", () => {
       }),
     ]);
     expect(body.providerApiPath).toBe("/api/generate");
+    expect(body.providerExecution).toEqual({
+      chatCapabilityDetected: false,
+      chatRequestAttempted: false,
+      chatRequestSucceeded: false,
+      generateRequestAttempted: true,
+    });
     expect(errorSpy).toHaveBeenCalledTimes(1);
     expect(errorSpy).toHaveBeenCalledWith(
       "[api/cases/start] error response",
@@ -176,6 +188,12 @@ describe("app/api/cases/start route", () => {
         analysisErrors: ["reconstruction: Required"],
         validationIssues: expect.any(Array),
         providerApiPath: "/api/generate",
+        providerExecution: {
+          chatCapabilityDetected: false,
+          chatRequestAttempted: false,
+          chatRequestSucceeded: false,
+          generateRequestAttempted: true,
+        },
         rawResponse,
       }),
     );

tests/graph/orchestrator.test.js

@@ -567,10 +567,17 @@ describe("lib/graph/orchestrator startCase", () => {
   });
 
   it("preserves an attempted provider API path on analysis failure", async () => {
+    const providerExecution = {
+      chatCapabilityDetected: true,
+      chatRequestAttempted: true,
+      chatRequestSucceeded: false,
+      generateRequestAttempted: true,
+    };
     mockAnalyseScenario.mockResolvedValue({
       success: false,
       error: "Provider failed",
       providerApiPath: "/api/chat",
+      providerExecution,
     });
     const { startCase } = await import("@/lib/graph/orchestrator.js");
 
@@ -580,6 +587,7 @@ describe("lib/graph/orchestrator startCase", () => {
       success: false,
       statusCode: 502,
       providerApiPath: "/api/chat",
+      providerExecution,
     });
   });
 

tests/llm/provider.test.js

@@ -1,4 +1,5 @@
 import { describe, expect, it, vi } from "vitest";
+import { __resetChatSupportForTests, getProvider } from "@/lib/llm/provider.js";
 
 describe("OllamaLlmProvider chat capability detection", () => {
   it("uses the configured model for the chat probe and keeps the chat path", async () => {
@@ -13,7 +14,7 @@ describe("OllamaLlmProvider chat capability detection", () => {
     process.env.OLLAMA_BASE_URL = "http://ollama.test";
 
     try {
-      const { getProvider } = await import("@/lib/llm/provider.js");
+      __resetChatSupportForTests();
       const result = await getProvider().generateReconstruction("prompt", "configured-model");
 
       expect(JSON.parse(fetchSpy.mock.calls[0][1].body)).toMatchObject({
@@ -37,6 +38,12 @@ describe("OllamaLlmProvider chat capability detection", () => {
       expect(result).toMatchObject({
         response: {},
         providerApiPath: "/api/chat",
+        providerExecution: {
+          chatCapabilityDetected: true,
+          chatRequestAttempted: true,
+          chatRequestSucceeded: true,
+          generateRequestAttempted: false,
+        },
       });
     } finally {
       vi.unstubAllGlobals();
@@ -45,7 +52,7 @@ describe("OllamaLlmProvider chat capability detection", () => {
     }
   });
 
-  it("reports the actually attempted generate path when generation fails", async () => {
+  it("reports chat-skipped generate fallback execution", async () => {
     const originalBaseUrl = process.env.OLLAMA_BASE_URL;
     const fetchSpy = vi.fn()
       .mockResolvedValueOnce({ ok: false, status: 501, body: { consume: vi.fn() } })
@@ -54,10 +61,18 @@ describe("OllamaLlmProvider chat capability detection", () => {
     process.env.OLLAMA_BASE_URL = "http://ollama.test";
 
     try {
-      const { getProvider } = await import("@/lib/llm/provider.js");
+      __resetChatSupportForTests();
       await expect(
         getProvider().generateReconstruction("prompt", "configured-model"),
-      ).rejects.toMatchObject({ providerApiPath: "/api/generate" });
+      ).rejects.toMatchObject({
+        providerApiPath: "/api/generate",
+        providerExecution: {
+          chatCapabilityDetected: false,
+          chatRequestAttempted: false,
+          chatRequestSucceeded: false,
+          generateRequestAttempted: true,
+        },
+      });
       expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/generate");
     } finally {
       vi.unstubAllGlobals();
@@ -65,4 +80,35 @@ describe("OllamaLlmProvider chat capability detection", () => {
       else process.env.OLLAMA_BASE_URL = originalBaseUrl;
     }
   });
+
+  it("reports chat-attempt-failed generate fallback execution", async () => {
+    const originalBaseUrl = process.env.OLLAMA_BASE_URL;
+    const fetchSpy = vi.fn()
+      .mockResolvedValueOnce({ ok: true, body: { consume: vi.fn() } })
+      .mockResolvedValueOnce({ ok: false, body: { consume: vi.fn() } })
+      .mockResolvedValueOnce({ ok: false, status: 500, text: async () => "failure" });
+    vi.stubGlobal("fetch", fetchSpy);
+    process.env.OLLAMA_BASE_URL = "http://ollama.test";
+
+    try {
+      __resetChatSupportForTests();
+      await expect(
+        getProvider().generateReconstruction("prompt", "configured-model"),
+      ).rejects.toMatchObject({
+        providerApiPath: "/api/generate",
+        providerExecution: {
+          chatCapabilityDetected: true,
+          chatRequestAttempted: true,
+          chatRequestSucceeded: false,
+          generateRequestAttempted: true,
+        },
+      });
+      expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/chat");
+      expect(fetchSpy.mock.calls[2][0]).toBe("http://ollama.test/api/generate");
+    } finally {
+      vi.unstubAllGlobals();
+      if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
+      else process.env.OLLAMA_BASE_URL = originalBaseUrl;
+    }
+  });
 });

tests/reconstruction/compatibility.test.js

@@ -115,6 +115,12 @@ describe("analyseScenario compatibility", () => {
   it("preserves an attempted provider API path on provider failure", async () => {
     const providerError = new Error("Provider failed");
     providerError.providerApiPath = "/api/chat";
+    providerError.providerExecution = {
+      chatCapabilityDetected: true,
+      chatRequestAttempted: true,
+      chatRequestSucceeded: false,
+      generateRequestAttempted: true,
+    };
     mockGenerateReconstruction.mockRejectedValue(providerError);
 
     const { analyseScenario } = await import("@/lib/analysis.js");
@@ -125,6 +131,7 @@ describe("analyseScenario compatibility", () => {
     expect(result).toMatchObject({
       success: false,
       providerApiPath: "/api/chat",
+      providerExecution: providerError.providerExecution,
     });
   });