fix(confidence-engine): expose failed provider path

app/api/cases/start/route.js

@@ -22,6 +22,7 @@ export async function POST(request) {
       validationErrors: result.validationErrors,
       analysisErrors: result.analysisErrors,
       validationIssues: result.validationIssues,
+      providerApiPath: result.providerApiPath,
       rawResponse: result.rawResponse ?? undefined,
     };
 
@@ -39,6 +40,7 @@ export async function POST(request) {
         diagnostics: result.diagnostics,
         analysisErrors: result.analysisErrors,
         validationIssues: result.validationIssues,
+        providerApiPath: result.providerApiPath,
         rawResponse: result.rawResponse ?? undefined,
       },
       { status },

docs/current-handoff.md

@@ -68,6 +68,11 @@
 - Direct POST `/api/chat` was proven supported. The false-negative cause was the capability probe using model `dummy-check`, which conflated model availability with endpoint capability.
 - The probe now uses the configured model; live behaviour after this fix remains untested.
 
+## Failed provider-path observability
+
+- The detector fix remains unverified live; a subsequent HTTP 500 left the actually attempted provider path unobservable.
+- Case-start failure diagnostics now preserve `providerApiPath` only when the provider reports an endpoint actually attempted during that request. The next step remains one production call.
+
 ## Current product architecture
 
 Three distinct routes, not a single page:

lib/analysis.js

@@ -81,6 +81,8 @@ export async function analyseScenario(scenario, opts = {}) {
     return buildErrorResponse(
       e.message || "Provider error during analysis",
       Date.now() - startTime,
+      "500",
+      e.providerApiPath,
     );
   }
 
@@ -154,7 +156,7 @@ function tryValidateAgainstSchema(data, schema) {
 
 // ── Result builders ──────────────────────────────────
 
-function buildErrorResponse(message, elapsed, statusCode = 500) {
+function buildErrorResponse(message, elapsed, statusCode = 500, providerApiPath) {
   return {
     success: false,
     error: message,
@@ -164,6 +166,7 @@ function buildErrorResponse(message, elapsed, statusCode = 500) {
     rawResponse: null,
     promptVersion: null,
     statusCode,
+    providerApiPath,
   };
 }
 

lib/graph/orchestrator.js

@@ -372,6 +372,7 @@ export async function startCase(body, dependencies = {}) {
       }),
       analysisErrors: analysis.errors ?? undefined,
       validationIssues: analysis.validationIssues ?? undefined,
+      providerApiPath: analysis.providerApiPath ?? undefined,
       rawResponse: analysis.rawResponse ?? undefined,
       statusCode: Number(analysis.statusCode) || 502,
     };

lib/llm/provider.js

@@ -119,7 +119,8 @@ class OllamaLlmProvider {
       try {
         const controller = new AbortController();
         const timeout = setTimeout(() => controller.abort(), 60000);
-        
+
+        apiUsed = "/api/chat";
         const res = await fetch(`${baseUrl}/api/chat`, {
           method: "POST",
           headers: { "Content-Type": "application/json" },
@@ -180,14 +181,14 @@ class OllamaLlmProvider {
         }
 
         fullResponseData = await res.json();
-        
+
         rawResponse = typeof fullResponseData.response === "string"
           ? fullResponseData.response
           : JSON.stringify(fullResponseData);
 
       } catch (e) {
         if (apiUsed === "/api/generate") {
-          throw new Error(
+          const error = new Error(
             `Ollama /api/generate request timed out after 5 minutes.\n\n` +
             `This usually means:\n` +
             `1. The model is loading into memory for the first time (cold start) — this can take several minutes\n` +
@@ -198,6 +199,8 @@ class OllamaLlmProvider {
             `- Use a smaller model (e.g., llama3.1 instead of llama3.1:70b)\n` +
             `- Check Ollama logs: \`ollama serve\` or look at your system logs`
           );
+          error.providerApiPath = apiUsed;
+          throw error;
         }
         throw e;
       }
@@ -225,7 +228,7 @@ class OllamaLlmProvider {
         }
       }
 
-      throw new Error(
+      const error = new Error(
         "Model produced empty output.\n\n" +
         "API used: " + (apiUsed || "none") + "\n" +
         "/api/chat supported: " + chatSupported + "\n" +
@@ -236,6 +239,8 @@ class OllamaLlmProvider {
         "- This Ollama version does not support format:json — using prompt instructions only (reliability varies)\n" +
         "- If your model is very small (e.g., tinyllama, phi), try a larger one like llama3.1 or mistral"
       );
+      error.providerApiPath = apiUsed;
+      throw error;
     }
 
     // ================================================================
@@ -245,7 +250,7 @@ class OllamaLlmProvider {
       return recoverJson(rawResponse);
     } catch (e) {
       if (e instanceof SyntaxError) {
-        throw new Error(
+        const error = new Error(
           "Model returned output that could not be parsed as valid JSON.\n\n" +
           "API used: " + (apiUsed || "none") + "\n" +
           "/api/chat supported: " + chatSupported + "\n" +
@@ -256,6 +261,8 @@ class OllamaLlmProvider {
           "- Shorten your scenario to under 500 words\n" +
           "- Consider upgrading Ollama: https://ollama.com/download"
         );
+        error.providerApiPath = apiUsed;
+        throw error;
       }
       throw e;
     }

tests/app/api/cases-start-route.test.js

@@ -138,6 +138,7 @@ describe("app/api/cases/start route", () => {
           received: "undefined",
         },
       ],
+      providerApiPath: "/api/generate",
       rawResponse,
     });
 
@@ -165,6 +166,7 @@ describe("app/api/cases/start route", () => {
         received: "undefined",
       }),
     ]);
+    expect(body.providerApiPath).toBe("/api/generate");
     expect(errorSpy).toHaveBeenCalledTimes(1);
     expect(errorSpy).toHaveBeenCalledWith(
       "[api/cases/start] error response",
@@ -173,6 +175,7 @@ describe("app/api/cases/start route", () => {
         error: "Provider unavailable",
         analysisErrors: ["reconstruction: Required"],
         validationIssues: expect.any(Array),
+        providerApiPath: "/api/generate",
         rawResponse,
       }),
     );

tests/graph/orchestrator.test.js

@@ -566,6 +566,23 @@ describe("lib/graph/orchestrator startCase", () => {
     expect(result.rawResponse).toHaveLength(2501);
   });
 
+  it("preserves an attempted provider API path on analysis failure", async () => {
+    mockAnalyseScenario.mockResolvedValue({
+      success: false,
+      error: "Provider failed",
+      providerApiPath: "/api/chat",
+    });
+    const { startCase } = await import("@/lib/graph/orchestrator.js");
+
+    const result = await startCase({ scenario: "Scenario text" });
+
+    expect(result).toMatchObject({
+      success: false,
+      statusCode: 502,
+      providerApiPath: "/api/chat",
+    });
+  });
+
   it("returns null selectedQuestion when neither analysis nor graph path yields a question", async () => {
     mockAnalyseScenario.mockResolvedValue(
       makeAnalysisResult({

tests/llm/provider.test.js

@@ -29,4 +29,25 @@ describe("OllamaLlmProvider chat capability detection", () => {
       else process.env.OLLAMA_BASE_URL = originalBaseUrl;
     }
   });
+
+  it("reports the actually attempted generate path when generation fails", async () => {
+    const originalBaseUrl = process.env.OLLAMA_BASE_URL;
+    const fetchSpy = vi.fn()
+      .mockResolvedValueOnce({ ok: false, status: 501, body: { consume: vi.fn() } })
+      .mockResolvedValueOnce({ ok: false, status: 500, text: async () => "failure" });
+    vi.stubGlobal("fetch", fetchSpy);
+    process.env.OLLAMA_BASE_URL = "http://ollama.test";
+
+    try {
+      const { getProvider } = await import("@/lib/llm/provider.js");
+      await expect(
+        getProvider().generateReconstruction("prompt", "configured-model"),
+      ).rejects.toMatchObject({ providerApiPath: "/api/generate" });
+      expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/generate");
+    } finally {
+      vi.unstubAllGlobals();
+      if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
+      else process.env.OLLAMA_BASE_URL = originalBaseUrl;
+    }
+  });
 });

tests/reconstruction/compatibility.test.js

@@ -112,6 +112,22 @@ describe("normaliseAnalysisResponse", () => {
 });
 
 describe("analyseScenario compatibility", () => {
+  it("preserves an attempted provider API path on provider failure", async () => {
+    const providerError = new Error("Provider failed");
+    providerError.providerApiPath = "/api/chat";
+    mockGenerateReconstruction.mockRejectedValue(providerError);
+
+    const { analyseScenario } = await import("@/lib/analysis.js");
+    const result = await analyseScenario("Scenario text", {
+      promptVersion: "v0.3",
+    });
+
+    expect(result).toMatchObject({
+      success: false,
+      providerApiPath: "/api/chat",
+    });
+  });
+
   it("succeeds when the only mismatch is null evidence source", async () => {
     mockGenerateReconstruction.mockResolvedValue({
       inputClassification: {