From bb3082d633bef445a3b189288a50ded215061ec0 Mon Sep 17 00:00:00 2001 From: robbond Date: Mon, 7 Sep 2026 09:35:53 +0100 Subject: [PATCH] experiment(confidence-engine): route UI journey provider centrally --- app/api/cases/overview/route.js | 4 +- app/api/cases/synthesis/route.js | 4 +- .../deconstruct/route.js | 4 +- docs/current-handoff.md | 8 +++- lib/analysis.js | 5 +-- lib/config.js | 24 ++++++++++- lib/graph/orchestrator.js | 11 ++--- lib/llm/provider.js | 7 +++ tests/app/api/cases-overview-route.test.js | 29 +++++++++++++ tests/app/api/cases-start-route.test.js | 2 + tests/app/api/cases-update-route.test.js | 2 + ...rent-understanding-synthesis-route.test.js | 2 + tests/focused-deconstruct-boundary.test.js | 7 ++- tests/graph/episode-reasoning-seam.test.js | 23 ++++++++++ tests/llm/provider.test.js | 43 +++++++++++++++++++ 15 files changed, 158 insertions(+), 17 deletions(-) create mode 100644 tests/app/api/cases-overview-route.test.js diff --git a/app/api/cases/overview/route.js b/app/api/cases/overview/route.js index 88ecfa6..66750ad 100644 --- a/app/api/cases/overview/route.js +++ b/app/api/cases/overview/route.js @@ -6,7 +6,7 @@ * Thin route pattern — no overview business logic here. */ -import { getProvider } from "@/lib/llm/provider.js"; +import { getProvider, getProviderModelName } from "@/lib/llm/provider.js"; import { synthesizeInvestigationOverview } from "@/lib/graph/investigation-overview-synthesis.js"; export async function POST(request) { @@ -33,7 +33,7 @@ export async function POST(request) { { situationGraph, findings, plausibleInterpretations }, { provider: getProvider(), - modelName: process.env.OLLAMA_MODEL ?? null, + modelName: getProviderModelName(), } ); diff --git a/app/api/cases/synthesis/route.js b/app/api/cases/synthesis/route.js index 712b102..9ad9601 100644 --- a/app/api/cases/synthesis/route.js +++ b/app/api/cases/synthesis/route.js @@ -9,7 +9,7 @@ * No synthesis business logic belongs in this file. */ -import { getProvider } from "@/lib/llm/provider.js"; +import { getProvider, getProviderModelName } from "@/lib/llm/provider.js"; import { synthesizeCurrentUnderstanding } from "@/lib/graph/current-understanding-synthesis.js"; export async function POST(request) { @@ -38,7 +38,7 @@ export async function POST(request) { { situationGraph, findings }, { provider: getProvider(), - modelName: process.env.OLLAMA_MODEL ?? null, + modelName: getProviderModelName(), } ); diff --git a/app/api/focused-investigation/deconstruct/route.js b/app/api/focused-investigation/deconstruct/route.js index 008fb2a..b84257c 100644 --- a/app/api/focused-investigation/deconstruct/route.js +++ b/app/api/focused-investigation/deconstruct/route.js @@ -1,4 +1,4 @@ -import { getProvider } from "@/lib/llm/provider"; +import { getProvider, getProviderModelName } from "@/lib/llm/provider"; import { buildFocusedDeconstructPrompt, focusedDeconstructJsonSchema, @@ -58,7 +58,7 @@ export async function POST(request) { const startedAt = Date.now(); const wrapper = await provider.generateReconstruction( prompt, - process.env.OLLAMA_MODEL, + getProviderModelName(), focusedDeconstructJsonSchema, ); const elapsedMs = Date.now() - startedAt; diff --git a/docs/current-handoff.md b/docs/current-handoff.md index 168ff53..e1de560 100644 --- a/docs/current-handoff.md +++ b/docs/current-handoff.md @@ -19,7 +19,7 @@ unless new end-to-end user-flow evidence reopens one of those boundaries. **Immediate next evidence question:** -Can the real browser UI run an entire investigation using OpenAI/Terra for every LLM stage, without any stage silently reverting to the default Ollama/Qwen provider? +Measure one real browser investigation configured for OpenAI/Terra across every LLM stage, including user-visible latency, call sequence, and cost. If YES, the next live experiment is one timed/costed OpenAI UI investigation measuring: - user-visible latency @@ -27,6 +27,12 @@ If YES, the next live experiment is one timed/costed OpenAI UI investigation mea - token usage where available - approximate cost per investigation +## Server-owned UI journey provider experiment + +- Server-only `CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER=openai` centrally resolves the existing OpenAI provider and `gpt-5.6-terra`; without it, normal production resolution remains Ollama/Qwen. An OpenAI key alone does not switch providers. +- Browser request contracts and client state remain unchanged. Deterministic coverage includes initial start, normal update, episode reconsideration, focused deconstruction, overview synthesis, and Current Understanding synthesis. +- Zero live calls occurred. Next boundary: one real Playwright-driven Terra investigation measuring user-visible latency, actual LLM-call sequence, and OpenAI usage/cost. + ## Repository checkpoint - **Branch:** `feature/initial-decomposition-v0.61` diff --git a/lib/analysis.js b/lib/analysis.js index cb5ae92..cb851ba 100644 --- a/lib/analysis.js +++ b/lib/analysis.js @@ -4,7 +4,7 @@ */ import { getConfig } from "../lib/config.js"; -import { getProvider } from "../lib/llm/provider.js"; +import { getProvider, getProviderModelName } from "../lib/llm/provider.js"; import { buildPrompt, PROMPT_VERSIONS, @@ -52,7 +52,6 @@ export async function analyseScenario(scenario, opts = {}) { return buildErrorResponse("Invalid server configuration", startTime, "500"); } - const { OLLAMA_BASE_URL: _ignored, OLLAMA_MODEL } = configResult.config; const promptVersion = opts.promptVersion || DEFAULT_PROMPT_VERSION; // ── Build prompt (experiment seam via env var bridge) ── @@ -73,7 +72,7 @@ export async function analyseScenario(scenario, opts = {}) { // ── Call provider ────────────────────────────────── const provider = opts.reconstructionProvider ?? getProvider(); - const reconstructionModelName = opts.reconstructionModelName ?? OLLAMA_MODEL; + const reconstructionModelName = opts.reconstructionModelName ?? getProviderModelName(); let rawResponse; let providerApiPath; let providerExecution; diff --git a/lib/config.js b/lib/config.js index ce302ed..02b0d41 100644 --- a/lib/config.js +++ b/lib/config.js @@ -5,7 +5,29 @@ const envSchema = z.object({ OLLAMA_MODEL: z.string().min(1), }); +const openAIExperimentEnvSchema = z.object({ + OPENAI_API_KEY: z.string().min(1), +}); + +export const OPENAI_UI_JOURNEY_EXPERIMENT_PROVIDER = "openai"; +export const OPENAI_TERRA_MODEL = "gpt-5.6-terra"; + +export function isOpenAIUiJourneyExperiment() { + return process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER === OPENAI_UI_JOURNEY_EXPERIMENT_PROVIDER; +} + export function getConfig() { + if (isOpenAIUiJourneyExperiment()) { + const parsed = openAIExperimentEnvSchema.safeParse({ + OPENAI_API_KEY: process.env.OPENAI_API_KEY, + }); + if (!parsed.success) return { ok: false, error: parsed.error.flatten().fieldErrors }; + return { + ok: true, + config: { provider: OPENAI_UI_JOURNEY_EXPERIMENT_PROVIDER, modelName: OPENAI_TERRA_MODEL }, + }; + } + const parsed = envSchema.safeParse({ OLLAMA_BASE_URL: process.env.OLLAMA_BASE_URL, OLLAMA_MODEL: process.env.OLLAMA_MODEL, @@ -15,7 +37,7 @@ export function getConfig() { return { ok: false, error: parsed.error.flatten().fieldErrors }; } - return { ok: true, config: parsed.data }; + return { ok: true, config: { ...parsed.data, provider: "ollama", modelName: parsed.data.OLLAMA_MODEL } }; } export function assertConfig() { diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index 1604d0f..cf42b88 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -5,7 +5,7 @@ import { analyseScenario } from "../analysis.js"; import { assertConfig } from "../config.js"; -import { getProvider } from "../llm/provider.js"; +import { getProvider, getProviderModelName } from "@/lib/llm/provider.js"; import { makeGraph, startCaseRequestSchema, @@ -666,8 +666,8 @@ async function updateCaseWithDependencies(body, dependencies = {}) { const startedAt = Date.now(); try { - const config = dependencies.config ?? assertConfig(); - modelName = config.OLLAMA_MODEL; + if (!dependencies.config) assertConfig(); + modelName = dependencies.modelName ?? dependencies.config?.modelName ?? dependencies.config?.OLLAMA_MODEL ?? getProviderModelName(); const prompt = buildPrompt({ situationGraph, @@ -1185,7 +1185,7 @@ export async function reconsiderCompletedEpisode(epiParams, deps = {}) { try { // Preserve original priority (deps.modelName > deps.config.OLLAMA_MODEL > getModelName) with assertConfig fallback only when no deps provide a model let resolvedConfig = undefined; - if (deps.modelName == null && deps.config?.OLLAMA_MODEL == null && deps.config !== null) { + if (deps.modelName == null && deps.config?.OLLAMA_MODEL == null && deps.config?.modelName == null && deps.config !== null) { const hasOtherDeps = Object.keys(deps).some((k) => k !== "config" && k !== "modelName"); if (hasOtherDeps) { resolvedConfig = undefined; // don't call assertConfig when deps is non-empty with other keys @@ -1196,8 +1196,9 @@ export async function reconsiderCompletedEpisode(epiParams, deps = {}) { const config = deps.config ?? resolvedConfig; modelName = deps.modelName ?? - (config != null ? config.OLLAMA_MODEL : undefined) ?? + (config != null ? (config.modelName ?? config.OLLAMA_MODEL) : undefined) ?? getModelName?.() ?? + getProviderModelName() ?? null; const promptBuilder = buildEpisodePrompt ?? buildEpisodeAwareGraphPrompt; diff --git a/lib/llm/provider.js b/lib/llm/provider.js index 50e8b9b..5c29249 100644 --- a/lib/llm/provider.js +++ b/lib/llm/provider.js @@ -1,5 +1,6 @@ import { z } from "zod"; import { reconstructionV2Schema } from "../reconstruction/schema.js"; +import { isOpenAIUiJourneyExperiment, OPENAI_TERRA_MODEL } from "../config.js"; /** * Provider abstraction — the app calls getProvider() which returns an object @@ -9,9 +10,15 @@ import { reconstructionV2Schema } from "../reconstruction/schema.js"; */ export function getProvider() { + if (isOpenAIUiJourneyExperiment()) return createOpenAIReconstructionProvider(); return new OllamaLlmProvider(); } +/** Server-owned model resolution for the configured application provider. */ +export function getProviderModelName() { + return isOpenAIUiJourneyExperiment() ? OPENAI_TERRA_MODEL : process.env.OLLAMA_MODEL ?? null; +} + /** * Experiment-only construction seam. Production provider selection remains Ollama. * @param {{ apiKey?: string, fetchImpl?: typeof fetch }} [options] diff --git a/tests/app/api/cases-overview-route.test.js b/tests/app/api/cases-overview-route.test.js new file mode 100644 index 0000000..0a63915 --- /dev/null +++ b/tests/app/api/cases-overview-route.test.js @@ -0,0 +1,29 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mockSynthesize = vi.fn(); + +vi.mock("@/lib/graph/investigation-overview-synthesis.js", () => ({ + synthesizeInvestigationOverview: (...args) => mockSynthesize(...args), +})); + +vi.mock("@/lib/llm/provider.js", () => ({ + getProvider: () => ({ generateReconstruction() {} }), + getProviderModelName: () => "gpt-5.6-terra", +})); + +describe("POST /api/cases/overview provider routing", () => { + beforeEach(() => mockSynthesize.mockClear()); + + it("passes the central provider and model resolution to overview synthesis", async () => { + mockSynthesize.mockResolvedValue({ understanding: "Understanding", plausibleInterpretations: "None" }); + const { POST } = await import("@/app/api/cases/overview/route.js"); + const response = await POST(new Request("http://localhost/api/cases/overview", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ situationGraph: { nodes: [], edges: [] }, findings: [], plausibleInterpretations: [] }), + })); + + expect(response.status).toBe(200); + expect(mockSynthesize.mock.calls[0][1]).toMatchObject({ modelName: "gpt-5.6-terra" }); + }); +}); \ No newline at end of file diff --git a/tests/app/api/cases-start-route.test.js b/tests/app/api/cases-start-route.test.js index 3dca41a..4bac8c8 100644 --- a/tests/app/api/cases-start-route.test.js +++ b/tests/app/api/cases-start-route.test.js @@ -39,6 +39,8 @@ describe("app/api/cases/start route", () => { await POST(request); expect(mockStartCase).toHaveBeenCalledWith({ scenario: "Scenario text" }); + expect(mockStartCase.mock.calls[0][0]).not.toHaveProperty("provider"); + expect(mockStartCase.mock.calls[0][0]).not.toHaveProperty("modelName"); }); it("returns 200 on success", async () => { diff --git a/tests/app/api/cases-update-route.test.js b/tests/app/api/cases-update-route.test.js index 07565b9..b899b30 100644 --- a/tests/app/api/cases-update-route.test.js +++ b/tests/app/api/cases-update-route.test.js @@ -72,6 +72,8 @@ describe("app/api/cases/update route", () => { ); expect(mockUpdateCase).toHaveBeenCalledWith(body, { applyProposal: true }); + expect(mockUpdateCase.mock.calls[0][0]).not.toHaveProperty("provider"); + expect(mockUpdateCase.mock.calls[0][0]).not.toHaveProperty("modelName"); }); it("invalid JSON returns 400", async () => { diff --git a/tests/app/api/current-understanding-synthesis-route.test.js b/tests/app/api/current-understanding-synthesis-route.test.js index 6ff9c53..bba0e97 100644 --- a/tests/app/api/current-understanding-synthesis-route.test.js +++ b/tests/app/api/current-understanding-synthesis-route.test.js @@ -10,6 +10,7 @@ vi.mock("@/lib/graph/current-understanding-synthesis.js", () => ({ vi.mock("@/lib/llm/provider.js", () => ({ getProvider: () => ({}), + getProviderModelName: () => "gpt-5.6-terra", })); // ── Helpers ───────────────────────────────────────────────── @@ -45,6 +46,7 @@ describe("POST /api/cases/synthesis — valid request", () => { expect(data.success).toBe(true); expect(data.currentUnderstanding).toBe("Synthesized result"); expect(mockSynthesize).toHaveBeenCalledTimes(1); + expect(mockSynthesize.mock.calls[0][1]).toMatchObject({ modelName: "gpt-5.6-terra" }); }); it("returns narrative result on success", async () => { diff --git a/tests/focused-deconstruct-boundary.test.js b/tests/focused-deconstruct-boundary.test.js index 66db5ee..3168712 100644 --- a/tests/focused-deconstruct-boundary.test.js +++ b/tests/focused-deconstruct-boundary.test.js @@ -51,6 +51,7 @@ describe("focused-deconstruct targetNodeId identity boundary", () => { vi.doMock("@/lib/llm/provider", () => ({ getProvider: () => makeMockProvider(inventedModelId), + getProviderModelName: () => "configured-model", })); const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js"); @@ -97,6 +98,7 @@ describe("focused-deconstruct targetNodeId identity boundary", () => { }); vi.doMock("@/lib/llm/provider", () => ({ getProvider: () => ({ generateReconstruction }), + getProviderModelName: () => "gpt-5.6-terra", })); const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js"); @@ -116,7 +118,7 @@ describe("focused-deconstruct targetNodeId identity boundary", () => { expect(response.status).toBe(200); expect(generateReconstruction).toHaveBeenCalledWith( expect.any(String), - process.env.OLLAMA_MODEL, + "gpt-5.6-terra", focusedDeconstructJsonSchema, ); }); @@ -149,6 +151,7 @@ describe("focused-deconstruct targetNodeId identity boundary", () => { providerExecution: { chatRequestAttempted: true }, }), }), + getProviderModelName: () => "configured-model", })); const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js"); @@ -228,6 +231,7 @@ describe("focused-deconstruct targetNodeId identity boundary", () => { providerExecution: { chatRequestAttempted: true }, }), }), + getProviderModelName: () => "configured-model", })); const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js"); @@ -291,6 +295,7 @@ describe("focused-deconstruct targetNodeId identity boundary", () => { providerExecution: { chatRequestAttempted: true, chatRequestSucceeded: true }, }), }), + getProviderModelName: () => "configured-model", })); const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js"); diff --git a/tests/graph/episode-reasoning-seam.test.js b/tests/graph/episode-reasoning-seam.test.js index 8c750e8..e4c5fe3 100644 --- a/tests/graph/episode-reasoning-seam.test.js +++ b/tests/graph/episode-reasoning-seam.test.js @@ -1,5 +1,10 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +vi.mock("@/lib/llm/provider.js", () => ({ + getProvider: () => ({ generateReconstruction() {} }), + getProviderModelName: () => "gpt-5.6-terra", +})); + // ── Fixtures ──────────────────────────────────────────────── function makePreparedEpisode(overrides = {}) { @@ -193,4 +198,22 @@ describe("episode reasoning seam (reconsiderCompletedEpisode)", () => { else process.env.OLLAMA_MODEL = prevModel; }); + it("uses the central provider model resolution when episode mode supplies no model override", async () => { + const provider = { + generateReconstruction: vi.fn().mockResolvedValue(makeProposalResponse()), + }; + const { reconsiderCompletedEpisode } = await import("@/lib/graph/orchestrator.js"); + + const result = await reconsiderCompletedEpisode( + { ...TEST_EPISODE, provider }, + { buildEpisodePrompt: () => "EPISODE_PROMPT" }, + ); + + expect(result.success).toBe(true); + expect(provider.generateReconstruction).toHaveBeenCalledWith( + "EPISODE_PROMPT", + "gpt-5.6-terra", + ); + }); + }); diff --git a/tests/llm/provider.test.js b/tests/llm/provider.test.js index 1a1c2b3..66a3d1a 100644 --- a/tests/llm/provider.test.js +++ b/tests/llm/provider.test.js @@ -4,6 +4,7 @@ import { createOpenAIStrictSchema, createOpenAIReconstructionProvider, getProvider, + getProviderModelName, normaliseOpenAITransportResponse, reconstructionJsonSchema, } from "@/lib/llm/provider.js"; @@ -11,6 +12,48 @@ import { reconstructionV2Schema } from "@/lib/reconstruction/schema.js"; import { z } from "zod"; describe("OllamaLlmProvider chat capability detection", () => { + it("keeps Ollama and its configured model when no experiment provider is selected", () => { + const previousExperimentProvider = process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER; + const previousModel = process.env.OLLAMA_MODEL; + const previousOpenAIKey = process.env.OPENAI_API_KEY; + delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER; + process.env.OLLAMA_MODEL = "qwen-default"; + process.env.OPENAI_API_KEY = "present-but-not-selected"; + + try { + expect(getProvider().constructor.name).toBe("OllamaLlmProvider"); + expect(getProviderModelName()).toBe("qwen-default"); + } finally { + if (previousExperimentProvider === undefined) delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER; + else process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER = previousExperimentProvider; + if (previousModel === undefined) delete process.env.OLLAMA_MODEL; + else process.env.OLLAMA_MODEL = previousModel; + if (previousOpenAIKey === undefined) delete process.env.OPENAI_API_KEY; + else process.env.OPENAI_API_KEY = previousOpenAIKey; + } + }); + + it("selects the existing OpenAI provider and Terra only for the explicit experiment configuration", () => { + const previousExperimentProvider = process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER; + const previousModel = process.env.OLLAMA_MODEL; + const previousOpenAIKey = process.env.OPENAI_API_KEY; + process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER = "openai"; + process.env.OPENAI_API_KEY = "test-key"; + process.env.OLLAMA_MODEL = "qwen-default"; + + try { + expect(getProvider().constructor.name).toBe("OpenAIReconstructionProvider"); + expect(getProviderModelName()).toBe("gpt-5.6-terra"); + } finally { + if (previousExperimentProvider === undefined) delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER; + else process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER = previousExperimentProvider; + if (previousModel === undefined) delete process.env.OLLAMA_MODEL; + else process.env.OLLAMA_MODEL = previousModel; + if (previousOpenAIKey === undefined) delete process.env.OPENAI_API_KEY; + else process.env.OPENAI_API_KEY = previousOpenAIKey; + } + }); + it("uses the configured model for the chat probe and keeps the chat path", async () => { const originalBaseUrl = process.env.OLLAMA_BASE_URL; const fetchSpy = vi.fn()