robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore(confidence-engine): expose reconstruction fallback path

Browse Source
This commit is contained in:
robbond
2026-09-05 19:13:39 +01:00
parent c7a0a79d0f
commit cd1c6f4fc5
9 changed files with 119 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/cases/start/route.js
+2
View File
@@ -23,6 +23,7 @@ export async function POST(request) {
analysisErrors: result.analysisErrors,
validationIssues: result.validationIssues,
providerApiPath: result.providerApiPath,
providerExecution: result.providerExecution,
rawResponse: result.rawResponse ?? undefined,
};
@@ -41,6 +42,7 @@ export async function POST(request) {
analysisErrors: result.analysisErrors,
validationIssues: result.validationIssues,
providerApiPath: result.providerApiPath,
providerExecution: result.providerExecution,
rawResponse: result.rawResponse ?? undefined,
},
{ status },
docs/current-handoff.md
+6
View File
@@ -86,6 +86,12 @@
- The deterministic provider-boundary test passes 2/2 under Node. A direct configured Ollama/Qwen `/api/chat` call accepted and obeyed the full schema in one call; `/api/generate`, chat detection, fallback, prompt, temperature, retries, and schema semantics are unchanged.
- Production repeatability remains untested after this change. Next restart point: a small repeated `/api/cases/start` stability observation using the fixed manufacturing scenario.
## Reconstruction fallback diagnostics
- Schema-constrained reconstruction `/api/chat` remains active. A later manual production 502 came through `/api/generate`; its unconstrained response omitted required transition confidence fields and failed Zod validation.
- Provider execution diagnostics now distinguish chat skipped due capability state, chat attempted and failed before generate fallback, and successful chat without fallback. They are deterministically verified through `/api/cases/start`; endpoint selection and fallback behaviour remain unchanged.
- Next restart point: one observation-only fixed-scenario production call to identify why `/api/generate` is reached.
## Current product architecture
Three distinct routes, not a single page:
lib/analysis.js
+8 -1
View File
@@ -73,6 +73,7 @@ export async function analyseScenario(scenario, opts = {}) {
const provider = getProvider();
let rawResponse;
let providerApiPath;
let providerExecution;
try {
const providerResult = await provider.generateReconstruction(
promptObj.prompt,
@@ -86,6 +87,7 @@ export async function analyseScenario(scenario, opts = {}) {
) {
rawResponse = providerResult.response;
providerApiPath = providerResult.providerApiPath;
providerExecution = providerResult.providerExecution;
} else {
rawResponse = providerResult;
}
@@ -95,6 +97,7 @@ export async function analyseScenario(scenario, opts = {}) {
Date.now() - startTime,
"500",
e.providerApiPath,
e.providerExecution,
);
}
@@ -150,6 +153,7 @@ export async function analyseScenario(scenario, opts = {}) {
promptVersion,
compatibility,
providerApiPath,
providerExecution,
);
}
@@ -169,7 +173,7 @@ function tryValidateAgainstSchema(data, schema) {
// ── Result builders ──────────────────────────────────
function buildErrorResponse(message, elapsed, statusCode = 500, providerApiPath) {
function buildErrorResponse(message, elapsed, statusCode = 500, providerApiPath, providerExecution) {
return {
success: false,
error: message,
@@ -180,6 +184,7 @@ function buildErrorResponse(message, elapsed, statusCode = 500, providerApiPath)
promptVersion: null,
statusCode,
providerApiPath,
providerExecution,
};
}
@@ -233,6 +238,7 @@ function buildPartialResult(
version,
compatibility,
providerApiPath,
providerExecution,
) {
let errors = [];
const validationIssues = error?.issues ?? [];
@@ -260,6 +266,7 @@ function buildPartialResult(
errors,
validationIssues,
providerApiPath,
providerExecution,
...buildCompatibilityDiagnostics(compatibility),
};
}
lib/graph/orchestrator.js
+1
View File
@@ -373,6 +373,7 @@ export async function startCase(body, dependencies = {}) {
analysisErrors: analysis.errors ?? undefined,
validationIssues: analysis.validationIssues ?? undefined,
providerApiPath: analysis.providerApiPath ?? undefined,
providerExecution: analysis.providerExecution ?? undefined,
rawResponse: analysis.rawResponse ?? undefined,
statusCode: Number(analysis.statusCode) || 502,
};
lib/llm/provider.js
+19
View File
@@ -64,6 +64,11 @@ function recoverJson(raw) {
let _chatSupported = null;
const reconstructionJsonSchema = z.toJSONSchema(reconstructionV2Schema);
/** @internal Test-only seam for isolated provider capability scenarios. */
export function __resetChatSupportForTests() {
_chatSupported = null;
}
async function detectChatSupport(baseUrl, modelName) {
if (_chatSupported !== null) return _chatSupported;
@@ -108,6 +113,12 @@ class OllamaLlmProvider {
let chatSupported = false;
let rawResponse = null;
let fullResponseData = null;
const providerExecution = {
chatCapabilityDetected: false,
chatRequestAttempted: false,
chatRequestSucceeded: false,
generateRequestAttempted: false,
};
// ================================================================
// Step 1: Detect whether /api/chat exists (cache result)
@@ -115,6 +126,7 @@ class OllamaLlmProvider {
try {
chatSupported = await detectChatSupport(baseUrl, modelName);
} catch { /* failed silently — defaults to false */ }
providerExecution.chatCapabilityDetected = chatSupported;
// ================================================================
// Step 2: Try /api/chat if supported with the reconstruction schema
@@ -125,6 +137,7 @@ class OllamaLlmProvider {
const timeout = setTimeout(() => controller.abort(), 60000);
apiUsed = "/api/chat";
providerExecution.chatRequestAttempted = true;
const res = await fetch(`${baseUrl}/api/chat`, {
method: "POST",
headers: { "Content-Type": "application/json" },
@@ -140,6 +153,7 @@ class OllamaLlmProvider {
clearTimeout(timeout);
if (res.ok) {
providerExecution.chatRequestSucceeded = true;
fullResponseData = await res.json();
rawResponse = typeof fullResponseData.message?.content === "string"
? fullResponseData.message.content
@@ -163,6 +177,7 @@ class OllamaLlmProvider {
const timeout = setTimeout(() => controller.abort(), 300000); // 5 min for cold start
apiUsed = "/api/generate"; // set BEFORE the request so we know which API failed
providerExecution.generateRequestAttempted = true;
const res = await fetch(`${baseUrl}/api/generate`, {
method: "POST",
@@ -204,6 +219,7 @@ class OllamaLlmProvider {
`- Check Ollama logs: \`ollama serve\` or look at your system logs`
);
error.providerApiPath = apiUsed;
error.providerExecution = providerExecution;
throw error;
}
throw e;
@@ -244,6 +260,7 @@ class OllamaLlmProvider {
"- If your model is very small (e.g., tinyllama, phi), try a larger one like llama3.1 or mistral"
);
error.providerApiPath = apiUsed;
error.providerExecution = providerExecution;
throw error;
}
@@ -254,6 +271,7 @@ class OllamaLlmProvider {
return {
response: recoverJson(rawResponse),
providerApiPath: apiUsed,
providerExecution,
};
} catch (e) {
if (e instanceof SyntaxError) {
@@ -269,6 +287,7 @@ class OllamaLlmProvider {
"- Consider upgrading Ollama: https://ollama.com/download"
);
error.providerApiPath = apiUsed;
error.providerExecution = providerExecution;
throw error;
}
throw e;
tests/app/api/cases-start-route.test.js
+18
View File
@@ -139,6 +139,12 @@ describe("app/api/cases/start route", () => {
},
],
providerApiPath: "/api/generate",
providerExecution: {
chatCapabilityDetected: false,
chatRequestAttempted: false,
chatRequestSucceeded: false,
generateRequestAttempted: true,
},
rawResponse,
});
@@ -167,6 +173,12 @@ describe("app/api/cases/start route", () => {
}),
]);
expect(body.providerApiPath).toBe("/api/generate");
expect(body.providerExecution).toEqual({
chatCapabilityDetected: false,
chatRequestAttempted: false,
chatRequestSucceeded: false,
generateRequestAttempted: true,
});
expect(errorSpy).toHaveBeenCalledTimes(1);
expect(errorSpy).toHaveBeenCalledWith(
"[api/cases/start] error response",
@@ -176,6 +188,12 @@ describe("app/api/cases/start route", () => {
analysisErrors: ["reconstruction: Required"],
validationIssues: expect.any(Array),
providerApiPath: "/api/generate",
providerExecution: {
chatCapabilityDetected: false,
chatRequestAttempted: false,
chatRequestSucceeded: false,
generateRequestAttempted: true,
},
rawResponse,
}),
);
tests/graph/orchestrator.test.js
+8
View File
@@ -567,10 +567,17 @@ describe("lib/graph/orchestrator startCase", () => {
});
it("preserves an attempted provider API path on analysis failure", async () => {
const providerExecution = {
chatCapabilityDetected: true,
chatRequestAttempted: true,
chatRequestSucceeded: false,
generateRequestAttempted: true,
};
mockAnalyseScenario.mockResolvedValue({
success: false,
error: "Provider failed",
providerApiPath: "/api/chat",
providerExecution,
});
const { startCase } = await import("@/lib/graph/orchestrator.js");
@@ -580,6 +587,7 @@ describe("lib/graph/orchestrator startCase", () => {
success: false,
statusCode: 502,
providerApiPath: "/api/chat",
providerExecution,
});
});
tests/llm/provider.test.js
+50 -4
View File
@@ -1,4 +1,5 @@
import { describe, expect, it, vi } from "vitest";
import { __resetChatSupportForTests, getProvider } from "@/lib/llm/provider.js";
describe("OllamaLlmProvider chat capability detection", () => {
it("uses the configured model for the chat probe and keeps the chat path", async () => {
@@ -13,7 +14,7 @@ describe("OllamaLlmProvider chat capability detection", () => {
process.env.OLLAMA_BASE_URL = "http://ollama.test";
try {
const { getProvider } = await import("@/lib/llm/provider.js");
__resetChatSupportForTests();
const result = await getProvider().generateReconstruction("prompt", "configured-model");
expect(JSON.parse(fetchSpy.mock.calls[0][1].body)).toMatchObject({
@@ -37,6 +38,12 @@ describe("OllamaLlmProvider chat capability detection", () => {
expect(result).toMatchObject({
response: {},
providerApiPath: "/api/chat",
providerExecution: {
chatCapabilityDetected: true,
chatRequestAttempted: true,
chatRequestSucceeded: true,
generateRequestAttempted: false,
},
});
} finally {
vi.unstubAllGlobals();
@@ -45,7 +52,7 @@ describe("OllamaLlmProvider chat capability detection", () => {
}
});
it("reports the actually attempted generate path when generation fails", async () => {
it("reports chat-skipped generate fallback execution", async () => {
const originalBaseUrl = process.env.OLLAMA_BASE_URL;
const fetchSpy = vi.fn()
.mockResolvedValueOnce({ ok: false, status: 501, body: { consume: vi.fn() } })
@@ -54,10 +61,18 @@ describe("OllamaLlmProvider chat capability detection", () => {
process.env.OLLAMA_BASE_URL = "http://ollama.test";
try {
const { getProvider } = await import("@/lib/llm/provider.js");
__resetChatSupportForTests();
await expect(
getProvider().generateReconstruction("prompt", "configured-model"),
).rejects.toMatchObject({ providerApiPath: "/api/generate" });
).rejects.toMatchObject({
providerApiPath: "/api/generate",
providerExecution: {
chatCapabilityDetected: false,
chatRequestAttempted: false,
chatRequestSucceeded: false,
generateRequestAttempted: true,
},
});
expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/generate");
} finally {
vi.unstubAllGlobals();
@@ -65,4 +80,35 @@ describe("OllamaLlmProvider chat capability detection", () => {
else process.env.OLLAMA_BASE_URL = originalBaseUrl;
}
});
it("reports chat-attempt-failed generate fallback execution", async () => {
const originalBaseUrl = process.env.OLLAMA_BASE_URL;
const fetchSpy = vi.fn()
.mockResolvedValueOnce({ ok: true, body: { consume: vi.fn() } })
.mockResolvedValueOnce({ ok: false, body: { consume: vi.fn() } })
.mockResolvedValueOnce({ ok: false, status: 500, text: async () => "failure" });
vi.stubGlobal("fetch", fetchSpy);
process.env.OLLAMA_BASE_URL = "http://ollama.test";
try {
__resetChatSupportForTests();
await expect(
getProvider().generateReconstruction("prompt", "configured-model"),
).rejects.toMatchObject({
providerApiPath: "/api/generate",
providerExecution: {
chatCapabilityDetected: true,
chatRequestAttempted: true,
chatRequestSucceeded: false,
generateRequestAttempted: true,
},
});
expect(fetchSpy.mock.calls[1][0]).toBe("http://ollama.test/api/chat");
expect(fetchSpy.mock.calls[2][0]).toBe("http://ollama.test/api/generate");
} finally {
vi.unstubAllGlobals();
if (originalBaseUrl === undefined) delete process.env.OLLAMA_BASE_URL;
else process.env.OLLAMA_BASE_URL = originalBaseUrl;
}
});
});
tests/reconstruction/compatibility.test.js
+7
View File
@@ -115,6 +115,12 @@ describe("analyseScenario compatibility", () => {
it("preserves an attempted provider API path on provider failure", async () => {
const providerError = new Error("Provider failed");
providerError.providerApiPath = "/api/chat";
providerError.providerExecution = {
chatCapabilityDetected: true,
chatRequestAttempted: true,
chatRequestSucceeded: false,
generateRequestAttempted: true,
};
mockGenerateReconstruction.mockRejectedValue(providerError);
const { analyseScenario } = await import("@/lib/analysis.js");
@@ -125,6 +131,7 @@ describe("analyseScenario compatibility", () => {
expect(result).toMatchObject({
success: false,
providerApiPath: "/api/chat",
providerExecution: providerError.providerExecution,
});
});