diff --git a/app/api/cases/start/route.js b/app/api/cases/start/route.js index 1a025f6..563fc22 100644 --- a/app/api/cases/start/route.js +++ b/app/api/cases/start/route.js @@ -22,6 +22,7 @@ export async function POST(request) { validationErrors: result.validationErrors, analysisErrors: result.analysisErrors, validationIssues: result.validationIssues, + providerApiPath: result.providerApiPath, rawResponse: result.rawResponse ?? undefined, }; @@ -39,6 +40,7 @@ export async function POST(request) { diagnostics: result.diagnostics, analysisErrors: result.analysisErrors, validationIssues: result.validationIssues, + providerApiPath: result.providerApiPath, rawResponse: result.rawResponse ?? undefined, }, { status }, diff --git a/docs/current-handoff.md b/docs/current-handoff.md index a79ac3f..81d0580 100644 --- a/docs/current-handoff.md +++ b/docs/current-handoff.md @@ -68,6 +68,11 @@ - Direct POST `/api/chat` was proven supported. The false-negative cause was the capability probe using model `dummy-check`, which conflated model availability with endpoint capability. - The probe now uses the configured model; live behaviour after this fix remains untested. +## Failed provider-path observability + +- The detector fix remains unverified live; a subsequent HTTP 500 left the actually attempted provider path unobservable. +- Case-start failure diagnostics now preserve `providerApiPath` only when the provider reports an endpoint actually attempted during that request. The next step remains one production call. + ## Current product architecture Three distinct routes, not a single page: diff --git a/lib/analysis.js b/lib/analysis.js index 945054d..8050cb4 100644 --- a/lib/analysis.js +++ b/lib/analysis.js @@ -81,6 +81,8 @@ export async function analyseScenario(scenario, opts = {}) { return buildErrorResponse( e.message || "Provider error during analysis", Date.now() - startTime, + "500", + e.providerApiPath, ); } @@ -154,7 +156,7 @@ function tryValidateAgainstSchema(data, schema) { // ── Result builders ────────────────────────────────── -function buildErrorResponse(message, elapsed, statusCode = 500) { +function buildErrorResponse(message, elapsed, statusCode = 500, providerApiPath) { return { success: false, error: message, @@ -164,6 +166,7 @@ function buildErrorResponse(message, elapsed, statusCode = 500) { rawResponse: null, promptVersion: null, statusCode, + providerApiPath, }; } diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index cf2eabc..09a877b 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -372,6 +372,7 @@ export async function startCase(body, dependencies = {}) { }), analysisErrors: analysis.errors ?? undefined, validationIssues: analysis.validationIssues ?? undefined, + providerApiPath: analysis.providerApiPath ?? undefined, rawResponse: analysis.rawResponse ?? undefined, statusCode: Number(analysis.statusCode) || 502, }; diff --git a/lib/llm/provider.js b/lib/llm/provider.js index ecea82e..3974de8 100644 --- a/lib/llm/provider.js +++ b/lib/llm/provider.js @@ -119,7 +119,8 @@ class OllamaLlmProvider { try { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 60000); - + + apiUsed = "/api/chat"; const res = await fetch(`${baseUrl}/api/chat`, { method: "POST", headers: { "Content-Type": "application/json" }, @@ -180,14 +181,14 @@ class OllamaLlmProvider { } fullResponseData = await res.json(); - + rawResponse = typeof fullResponseData.response === "string" ? fullResponseData.response : JSON.stringify(fullResponseData); } catch (e) { if (apiUsed === "/api/generate") { - throw new Error( + const error = new Error( `Ollama /api/generate request timed out after 5 minutes.\n\n` + `This usually means:\n` + `1. The model is loading into memory for the first time (cold start) — this can take several minutes\n` + @@ -198,6 +199,8 @@ class OllamaLlmProvider { `- Use a smaller model (e.g., llama3.1 instead of llama3.1:70b)\n` + `- Check Ollama logs: \`ollama serve\` or look at your system logs` ); + error.providerApiPath = apiUsed; + throw error; } throw e; } @@ -225,7 +228,7 @@ class OllamaLlmProvider { } } - throw new Error( + const error = new Error( "Model produced empty output.\n\n" + "API used: " + (apiUsed || "none") + "\n" + "/api/chat supported: " + chatSupported + "\n" + @@ -236,6 +239,8 @@ class OllamaLlmProvider { "- This Ollama version does not support format:json — using prompt instructions only (reliability varies)\n" + "- If your model is very small (e.g., tinyllama, phi), try a larger one like llama3.1 or mistral" ); + error.providerApiPath = apiUsed; + throw error; } // ================================================================ @@ -245,7 +250,7 @@ class OllamaLlmProvider { return recoverJson(rawResponse); } catch (e) { if (e instanceof SyntaxError) { - throw new Error( + const error = new Error( "Model returned output that could not be parsed as valid JSON.\n\n" + "API used: " + (apiUsed || "none") + "\n" + "/api/chat supported: " + chatSupported + "\n" + @@ -256,6 +261,8 @@ class OllamaLlmProvider { "- Shorten your scenario to under 500 words\n" + "- Consider upgrading Ollama: https://ollama.com/download" ); + error.providerApiPath = apiUsed; + throw error; } throw e; } diff --git a/tests/app/api/cases-start-route.test.js b/tests/app/api/cases-start-route.test.js index 4cfcd86..84930a0 100644 --- a/tests/app/api/cases-start-route.test.js +++ b/tests/app/api/cases-start-route.test.js @@ -138,6 +138,7 @@ describe("app/api/cases/start route", () => { received: "undefined", }, ], + providerApiPath: "/api/generate", rawResponse, }); @@ -165,6 +166,7 @@ describe("app/api/cases/start route", () => { received: "undefined", }), ]); + expect(body.providerApiPath).toBe("/api/generate"); expect(errorSpy).toHaveBeenCalledTimes(1); expect(errorSpy).toHaveBeenCalledWith( "[api/cases/start] error response", @@ -173,6 +175,7 @@ describe("app/api/cases/start route", () => { error: "Provider unavailable", analysisErrors: ["reconstruction: Required"], validationIssues: expect.any(Array), + providerApiPath: "/api/generate", rawResponse, }), ); diff --git a/tests/graph/orchestrator.test.js b/tests/graph/orchestrator.test.js index ce68038..5fb499a 100644 --- a/tests/graph/orchestrator.test.js +++ b/tests/graph/orchestrator.test.js @@ -566,6 +566,23 @@ describe("lib/graph/orchestrator startCase", () => { expect(result.rawResponse).toHaveLength(2501); }); + it("preserves an attempted provider API path on analysis failure", async () => { + mockAnalyseScenario.mockResolvedValue({ + success: false, + error: "Provider failed", + providerApiPath: "/api/chat", + }); + const { startCase } = await import("@/lib/graph/orchestrator.js"); + + const result = await startCase({ scenario: "Scenario text" }); + + expect(result).toMatchObject({ + success: false, + statusCode: 502, + providerApiPath: "/api/chat", + }); + }); + it("returns null selectedQuestion when neither analysis nor graph path yields a question", async () => { mockAnalyseScenario.mockResolvedValue( makeAnalysisResult({ diff --git a/tests/llm/provider.test.js b/tests/llm/provider.test.js index 289fd29..98cfc29 100644 --- a/tests/llm/provider.test.js +++ b/tests/llm/provider.test.js @@ -29,4 +29,25 @@ describe("OllamaLlmProvider chat capability detection", () => { else process.env.OLLAMA_BASE_URL = originalBaseUrl; } }); + + it("reports the actually attempted generate path when generation fails", async () => { + const originalBaseUrl = process.env.OLLAMA_BASE_URL; + const fetchSpy = vi.fn() + .mockResolvedValueOnce({ ok: false, status: 501, body: { consume: vi.fn() } }) + .mockResolvedValueOnce({ ok: false, status: 500, text: async () => "failure" }); + vi.stubGlobal("fetch", fetchSpy); + process.env.OLLAMA_BASE_URL = "http://ollama.test"; + + try { + const { getProvider } = await import("@/lib/llm/provider.js"); + await expect( + getProvider().generateReconstruction("prompt", "configured-model"), + ).rejects.toMatchObject({ providerApiPath: "/api/generate" }); + expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/generate"); + } finally { + vi.unstubAllGlobals(); + if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL; + else process.env.OLLAMA_BASE_URL = originalBaseUrl; + } + }); }); \ No newline at end of file diff --git a/tests/reconstruction/compatibility.test.js b/tests/reconstruction/compatibility.test.js index 1c59490..fe48086 100644 --- a/tests/reconstruction/compatibility.test.js +++ b/tests/reconstruction/compatibility.test.js @@ -112,6 +112,22 @@ describe("normaliseAnalysisResponse", () => { }); describe("analyseScenario compatibility", () => { + it("preserves an attempted provider API path on provider failure", async () => { + const providerError = new Error("Provider failed"); + providerError.providerApiPath = "/api/chat"; + mockGenerateReconstruction.mockRejectedValue(providerError); + + const { analyseScenario } = await import("@/lib/analysis.js"); + const result = await analyseScenario("Scenario text", { + promptVersion: "v0.3", + }); + + expect(result).toMatchObject({ + success: false, + providerApiPath: "/api/chat", + }); + }); + it("succeeds when the only mismatch is null evidence source", async () => { mockGenerateReconstruction.mockResolvedValue({ inputClassification: {