experiment(confidence-engine): route UI journey provider centrally

This commit is contained in:
2026-09-07 09:35:53 +01:00
parent 642a969b18
commit bb3082d633
15 changed files with 158 additions and 17 deletions
+2 -2
View File
@@ -6,7 +6,7 @@
* Thin route pattern — no overview business logic here. * Thin route pattern — no overview business logic here.
*/ */
import { getProvider } from "@/lib/llm/provider.js"; import { getProvider, getProviderModelName } from "@/lib/llm/provider.js";
import { synthesizeInvestigationOverview } from "@/lib/graph/investigation-overview-synthesis.js"; import { synthesizeInvestigationOverview } from "@/lib/graph/investigation-overview-synthesis.js";
export async function POST(request) { export async function POST(request) {
@@ -33,7 +33,7 @@ export async function POST(request) {
{ situationGraph, findings, plausibleInterpretations }, { situationGraph, findings, plausibleInterpretations },
{ {
provider: getProvider(), provider: getProvider(),
modelName: process.env.OLLAMA_MODEL ?? null, modelName: getProviderModelName(),
} }
); );
+2 -2
View File
@@ -9,7 +9,7 @@
* No synthesis business logic belongs in this file. * No synthesis business logic belongs in this file.
*/ */
import { getProvider } from "@/lib/llm/provider.js"; import { getProvider, getProviderModelName } from "@/lib/llm/provider.js";
import { synthesizeCurrentUnderstanding } from "@/lib/graph/current-understanding-synthesis.js"; import { synthesizeCurrentUnderstanding } from "@/lib/graph/current-understanding-synthesis.js";
export async function POST(request) { export async function POST(request) {
@@ -38,7 +38,7 @@ export async function POST(request) {
{ situationGraph, findings }, { situationGraph, findings },
{ {
provider: getProvider(), provider: getProvider(),
modelName: process.env.OLLAMA_MODEL ?? null, modelName: getProviderModelName(),
} }
); );
@@ -1,4 +1,4 @@
import { getProvider } from "@/lib/llm/provider"; import { getProvider, getProviderModelName } from "@/lib/llm/provider";
import { import {
buildFocusedDeconstructPrompt, buildFocusedDeconstructPrompt,
focusedDeconstructJsonSchema, focusedDeconstructJsonSchema,
@@ -58,7 +58,7 @@ export async function POST(request) {
const startedAt = Date.now(); const startedAt = Date.now();
const wrapper = await provider.generateReconstruction( const wrapper = await provider.generateReconstruction(
prompt, prompt,
process.env.OLLAMA_MODEL, getProviderModelName(),
focusedDeconstructJsonSchema, focusedDeconstructJsonSchema,
); );
const elapsedMs = Date.now() - startedAt; const elapsedMs = Date.now() - startedAt;
+7 -1
View File
@@ -19,7 +19,7 @@ unless new end-to-end user-flow evidence reopens one of those boundaries.
**Immediate next evidence question:** **Immediate next evidence question:**
Can the real browser UI run an entire investigation using OpenAI/Terra for every LLM stage, without any stage silently reverting to the default Ollama/Qwen provider? Measure one real browser investigation configured for OpenAI/Terra across every LLM stage, including user-visible latency, call sequence, and cost.
If YES, the next live experiment is one timed/costed OpenAI UI investigation measuring: If YES, the next live experiment is one timed/costed OpenAI UI investigation measuring:
- user-visible latency - user-visible latency
@@ -27,6 +27,12 @@ If YES, the next live experiment is one timed/costed OpenAI UI investigation mea
- token usage where available - token usage where available
- approximate cost per investigation - approximate cost per investigation
## Server-owned UI journey provider experiment
- Server-only `CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER=openai` centrally resolves the existing OpenAI provider and `gpt-5.6-terra`; without it, normal production resolution remains Ollama/Qwen. An OpenAI key alone does not switch providers.
- Browser request contracts and client state remain unchanged. Deterministic coverage includes initial start, normal update, episode reconsideration, focused deconstruction, overview synthesis, and Current Understanding synthesis.
- Zero live calls occurred. Next boundary: one real Playwright-driven Terra investigation measuring user-visible latency, actual LLM-call sequence, and OpenAI usage/cost.
## Repository checkpoint ## Repository checkpoint
- **Branch:** `feature/initial-decomposition-v0.61` - **Branch:** `feature/initial-decomposition-v0.61`
+2 -3
View File
@@ -4,7 +4,7 @@
*/ */
import { getConfig } from "../lib/config.js"; import { getConfig } from "../lib/config.js";
import { getProvider } from "../lib/llm/provider.js"; import { getProvider, getProviderModelName } from "../lib/llm/provider.js";
import { import {
buildPrompt, buildPrompt,
PROMPT_VERSIONS, PROMPT_VERSIONS,
@@ -52,7 +52,6 @@ export async function analyseScenario(scenario, opts = {}) {
return buildErrorResponse("Invalid server configuration", startTime, "500"); return buildErrorResponse("Invalid server configuration", startTime, "500");
} }
const { OLLAMA_BASE_URL: _ignored, OLLAMA_MODEL } = configResult.config;
const promptVersion = opts.promptVersion || DEFAULT_PROMPT_VERSION; const promptVersion = opts.promptVersion || DEFAULT_PROMPT_VERSION;
// ── Build prompt (experiment seam via env var bridge) ── // ── Build prompt (experiment seam via env var bridge) ──
@@ -73,7 +72,7 @@ export async function analyseScenario(scenario, opts = {}) {
// ── Call provider ────────────────────────────────── // ── Call provider ──────────────────────────────────
const provider = opts.reconstructionProvider ?? getProvider(); const provider = opts.reconstructionProvider ?? getProvider();
const reconstructionModelName = opts.reconstructionModelName ?? OLLAMA_MODEL; const reconstructionModelName = opts.reconstructionModelName ?? getProviderModelName();
let rawResponse; let rawResponse;
let providerApiPath; let providerApiPath;
let providerExecution; let providerExecution;
+23 -1
View File
@@ -5,7 +5,29 @@ const envSchema = z.object({
OLLAMA_MODEL: z.string().min(1), OLLAMA_MODEL: z.string().min(1),
}); });
const openAIExperimentEnvSchema = z.object({
OPENAI_API_KEY: z.string().min(1),
});
export const OPENAI_UI_JOURNEY_EXPERIMENT_PROVIDER = "openai";
export const OPENAI_TERRA_MODEL = "gpt-5.6-terra";
export function isOpenAIUiJourneyExperiment() {
return process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER === OPENAI_UI_JOURNEY_EXPERIMENT_PROVIDER;
}
export function getConfig() { export function getConfig() {
if (isOpenAIUiJourneyExperiment()) {
const parsed = openAIExperimentEnvSchema.safeParse({
OPENAI_API_KEY: process.env.OPENAI_API_KEY,
});
if (!parsed.success) return { ok: false, error: parsed.error.flatten().fieldErrors };
return {
ok: true,
config: { provider: OPENAI_UI_JOURNEY_EXPERIMENT_PROVIDER, modelName: OPENAI_TERRA_MODEL },
};
}
const parsed = envSchema.safeParse({ const parsed = envSchema.safeParse({
OLLAMA_BASE_URL: process.env.OLLAMA_BASE_URL, OLLAMA_BASE_URL: process.env.OLLAMA_BASE_URL,
OLLAMA_MODEL: process.env.OLLAMA_MODEL, OLLAMA_MODEL: process.env.OLLAMA_MODEL,
@@ -15,7 +37,7 @@ export function getConfig() {
return { ok: false, error: parsed.error.flatten().fieldErrors }; return { ok: false, error: parsed.error.flatten().fieldErrors };
} }
return { ok: true, config: parsed.data }; return { ok: true, config: { ...parsed.data, provider: "ollama", modelName: parsed.data.OLLAMA_MODEL } };
} }
export function assertConfig() { export function assertConfig() {
+6 -5
View File
@@ -5,7 +5,7 @@
import { analyseScenario } from "../analysis.js"; import { analyseScenario } from "../analysis.js";
import { assertConfig } from "../config.js"; import { assertConfig } from "../config.js";
import { getProvider } from "../llm/provider.js"; import { getProvider, getProviderModelName } from "@/lib/llm/provider.js";
import { import {
makeGraph, makeGraph,
startCaseRequestSchema, startCaseRequestSchema,
@@ -666,8 +666,8 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
const startedAt = Date.now(); const startedAt = Date.now();
try { try {
const config = dependencies.config ?? assertConfig(); if (!dependencies.config) assertConfig();
modelName = config.OLLAMA_MODEL; modelName = dependencies.modelName ?? dependencies.config?.modelName ?? dependencies.config?.OLLAMA_MODEL ?? getProviderModelName();
const prompt = buildPrompt({ const prompt = buildPrompt({
situationGraph, situationGraph,
@@ -1185,7 +1185,7 @@ export async function reconsiderCompletedEpisode(epiParams, deps = {}) {
try { try {
// Preserve original priority (deps.modelName > deps.config.OLLAMA_MODEL > getModelName) with assertConfig fallback only when no deps provide a model // Preserve original priority (deps.modelName > deps.config.OLLAMA_MODEL > getModelName) with assertConfig fallback only when no deps provide a model
let resolvedConfig = undefined; let resolvedConfig = undefined;
if (deps.modelName == null && deps.config?.OLLAMA_MODEL == null && deps.config !== null) { if (deps.modelName == null && deps.config?.OLLAMA_MODEL == null && deps.config?.modelName == null && deps.config !== null) {
const hasOtherDeps = Object.keys(deps).some((k) => k !== "config" && k !== "modelName"); const hasOtherDeps = Object.keys(deps).some((k) => k !== "config" && k !== "modelName");
if (hasOtherDeps) { if (hasOtherDeps) {
resolvedConfig = undefined; // don't call assertConfig when deps is non-empty with other keys resolvedConfig = undefined; // don't call assertConfig when deps is non-empty with other keys
@@ -1196,8 +1196,9 @@ export async function reconsiderCompletedEpisode(epiParams, deps = {}) {
const config = deps.config ?? resolvedConfig; const config = deps.config ?? resolvedConfig;
modelName = modelName =
deps.modelName ?? deps.modelName ??
(config != null ? config.OLLAMA_MODEL : undefined) ?? (config != null ? (config.modelName ?? config.OLLAMA_MODEL) : undefined) ??
getModelName?.() ?? getModelName?.() ??
getProviderModelName() ??
null; null;
const promptBuilder = buildEpisodePrompt ?? buildEpisodeAwareGraphPrompt; const promptBuilder = buildEpisodePrompt ?? buildEpisodeAwareGraphPrompt;
+7
View File
@@ -1,5 +1,6 @@
import { z } from "zod"; import { z } from "zod";
import { reconstructionV2Schema } from "../reconstruction/schema.js"; import { reconstructionV2Schema } from "../reconstruction/schema.js";
import { isOpenAIUiJourneyExperiment, OPENAI_TERRA_MODEL } from "../config.js";
/** /**
* Provider abstraction — the app calls getProvider() which returns an object * Provider abstraction — the app calls getProvider() which returns an object
@@ -9,9 +10,15 @@ import { reconstructionV2Schema } from "../reconstruction/schema.js";
*/ */
export function getProvider() { export function getProvider() {
if (isOpenAIUiJourneyExperiment()) return createOpenAIReconstructionProvider();
return new OllamaLlmProvider(); return new OllamaLlmProvider();
} }
/** Server-owned model resolution for the configured application provider. */
export function getProviderModelName() {
return isOpenAIUiJourneyExperiment() ? OPENAI_TERRA_MODEL : process.env.OLLAMA_MODEL ?? null;
}
/** /**
* Experiment-only construction seam. Production provider selection remains Ollama. * Experiment-only construction seam. Production provider selection remains Ollama.
* @param {{ apiKey?: string, fetchImpl?: typeof fetch }} [options] * @param {{ apiKey?: string, fetchImpl?: typeof fetch }} [options]
@@ -0,0 +1,29 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const mockSynthesize = vi.fn();
vi.mock("@/lib/graph/investigation-overview-synthesis.js", () => ({
synthesizeInvestigationOverview: (...args) => mockSynthesize(...args),
}));
vi.mock("@/lib/llm/provider.js", () => ({
getProvider: () => ({ generateReconstruction() {} }),
getProviderModelName: () => "gpt-5.6-terra",
}));
describe("POST /api/cases/overview provider routing", () => {
beforeEach(() => mockSynthesize.mockClear());
it("passes the central provider and model resolution to overview synthesis", async () => {
mockSynthesize.mockResolvedValue({ understanding: "Understanding", plausibleInterpretations: "None" });
const { POST } = await import("@/app/api/cases/overview/route.js");
const response = await POST(new Request("http://localhost/api/cases/overview", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ situationGraph: { nodes: [], edges: [] }, findings: [], plausibleInterpretations: [] }),
}));
expect(response.status).toBe(200);
expect(mockSynthesize.mock.calls[0][1]).toMatchObject({ modelName: "gpt-5.6-terra" });
});
});
+2
View File
@@ -39,6 +39,8 @@ describe("app/api/cases/start route", () => {
await POST(request); await POST(request);
expect(mockStartCase).toHaveBeenCalledWith({ scenario: "Scenario text" }); expect(mockStartCase).toHaveBeenCalledWith({ scenario: "Scenario text" });
expect(mockStartCase.mock.calls[0][0]).not.toHaveProperty("provider");
expect(mockStartCase.mock.calls[0][0]).not.toHaveProperty("modelName");
}); });
it("returns 200 on success", async () => { it("returns 200 on success", async () => {
+2
View File
@@ -72,6 +72,8 @@ describe("app/api/cases/update route", () => {
); );
expect(mockUpdateCase).toHaveBeenCalledWith(body, { applyProposal: true }); expect(mockUpdateCase).toHaveBeenCalledWith(body, { applyProposal: true });
expect(mockUpdateCase.mock.calls[0][0]).not.toHaveProperty("provider");
expect(mockUpdateCase.mock.calls[0][0]).not.toHaveProperty("modelName");
}); });
it("invalid JSON returns 400", async () => { it("invalid JSON returns 400", async () => {
@@ -10,6 +10,7 @@ vi.mock("@/lib/graph/current-understanding-synthesis.js", () => ({
vi.mock("@/lib/llm/provider.js", () => ({ vi.mock("@/lib/llm/provider.js", () => ({
getProvider: () => ({}), getProvider: () => ({}),
getProviderModelName: () => "gpt-5.6-terra",
})); }));
// ── Helpers ───────────────────────────────────────────────── // ── Helpers ─────────────────────────────────────────────────
@@ -45,6 +46,7 @@ describe("POST /api/cases/synthesis — valid request", () => {
expect(data.success).toBe(true); expect(data.success).toBe(true);
expect(data.currentUnderstanding).toBe("Synthesized result"); expect(data.currentUnderstanding).toBe("Synthesized result");
expect(mockSynthesize).toHaveBeenCalledTimes(1); expect(mockSynthesize).toHaveBeenCalledTimes(1);
expect(mockSynthesize.mock.calls[0][1]).toMatchObject({ modelName: "gpt-5.6-terra" });
}); });
it("returns narrative result on success", async () => { it("returns narrative result on success", async () => {
+6 -1
View File
@@ -51,6 +51,7 @@ describe("focused-deconstruct targetNodeId identity boundary", () => {
vi.doMock("@/lib/llm/provider", () => ({ vi.doMock("@/lib/llm/provider", () => ({
getProvider: () => makeMockProvider(inventedModelId), getProvider: () => makeMockProvider(inventedModelId),
getProviderModelName: () => "configured-model",
})); }));
const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js"); const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js");
@@ -97,6 +98,7 @@ describe("focused-deconstruct targetNodeId identity boundary", () => {
}); });
vi.doMock("@/lib/llm/provider", () => ({ vi.doMock("@/lib/llm/provider", () => ({
getProvider: () => ({ generateReconstruction }), getProvider: () => ({ generateReconstruction }),
getProviderModelName: () => "gpt-5.6-terra",
})); }));
const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js"); const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js");
@@ -116,7 +118,7 @@ describe("focused-deconstruct targetNodeId identity boundary", () => {
expect(response.status).toBe(200); expect(response.status).toBe(200);
expect(generateReconstruction).toHaveBeenCalledWith( expect(generateReconstruction).toHaveBeenCalledWith(
expect.any(String), expect.any(String),
process.env.OLLAMA_MODEL, "gpt-5.6-terra",
focusedDeconstructJsonSchema, focusedDeconstructJsonSchema,
); );
}); });
@@ -149,6 +151,7 @@ describe("focused-deconstruct targetNodeId identity boundary", () => {
providerExecution: { chatRequestAttempted: true }, providerExecution: { chatRequestAttempted: true },
}), }),
}), }),
getProviderModelName: () => "configured-model",
})); }));
const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js"); const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js");
@@ -228,6 +231,7 @@ describe("focused-deconstruct targetNodeId identity boundary", () => {
providerExecution: { chatRequestAttempted: true }, providerExecution: { chatRequestAttempted: true },
}), }),
}), }),
getProviderModelName: () => "configured-model",
})); }));
const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js"); const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js");
@@ -291,6 +295,7 @@ describe("focused-deconstruct targetNodeId identity boundary", () => {
providerExecution: { chatRequestAttempted: true, chatRequestSucceeded: true }, providerExecution: { chatRequestAttempted: true, chatRequestSucceeded: true },
}), }),
}), }),
getProviderModelName: () => "configured-model",
})); }));
const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js"); const { POST } = await import("../app/api/focused-investigation/deconstruct/route.js");
@@ -1,5 +1,10 @@
import { beforeEach, describe, expect, it, vi } from "vitest"; import { beforeEach, describe, expect, it, vi } from "vitest";
vi.mock("@/lib/llm/provider.js", () => ({
getProvider: () => ({ generateReconstruction() {} }),
getProviderModelName: () => "gpt-5.6-terra",
}));
// ── Fixtures ──────────────────────────────────────────────── // ── Fixtures ────────────────────────────────────────────────
function makePreparedEpisode(overrides = {}) { function makePreparedEpisode(overrides = {}) {
@@ -193,4 +198,22 @@ describe("episode reasoning seam (reconsiderCompletedEpisode)", () => {
else process.env.OLLAMA_MODEL = prevModel; else process.env.OLLAMA_MODEL = prevModel;
}); });
it("uses the central provider model resolution when episode mode supplies no model override", async () => {
const provider = {
generateReconstruction: vi.fn().mockResolvedValue(makeProposalResponse()),
};
const { reconsiderCompletedEpisode } = await import("@/lib/graph/orchestrator.js");
const result = await reconsiderCompletedEpisode(
{ ...TEST_EPISODE, provider },
{ buildEpisodePrompt: () => "EPISODE_PROMPT" },
);
expect(result.success).toBe(true);
expect(provider.generateReconstruction).toHaveBeenCalledWith(
"EPISODE_PROMPT",
"gpt-5.6-terra",
);
});
}); });
+43
View File
@@ -4,6 +4,7 @@ import {
createOpenAIStrictSchema, createOpenAIStrictSchema,
createOpenAIReconstructionProvider, createOpenAIReconstructionProvider,
getProvider, getProvider,
getProviderModelName,
normaliseOpenAITransportResponse, normaliseOpenAITransportResponse,
reconstructionJsonSchema, reconstructionJsonSchema,
} from "@/lib/llm/provider.js"; } from "@/lib/llm/provider.js";
@@ -11,6 +12,48 @@ import { reconstructionV2Schema } from "@/lib/reconstruction/schema.js";
import { z } from "zod"; import { z } from "zod";
describe("OllamaLlmProvider chat capability detection", () => { describe("OllamaLlmProvider chat capability detection", () => {
it("keeps Ollama and its configured model when no experiment provider is selected", () => {
const previousExperimentProvider = process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER;
const previousModel = process.env.OLLAMA_MODEL;
const previousOpenAIKey = process.env.OPENAI_API_KEY;
delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER;
process.env.OLLAMA_MODEL = "qwen-default";
process.env.OPENAI_API_KEY = "present-but-not-selected";
try {
expect(getProvider().constructor.name).toBe("OllamaLlmProvider");
expect(getProviderModelName()).toBe("qwen-default");
} finally {
if (previousExperimentProvider === undefined) delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER;
else process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER = previousExperimentProvider;
if (previousModel === undefined) delete process.env.OLLAMA_MODEL;
else process.env.OLLAMA_MODEL = previousModel;
if (previousOpenAIKey === undefined) delete process.env.OPENAI_API_KEY;
else process.env.OPENAI_API_KEY = previousOpenAIKey;
}
});
it("selects the existing OpenAI provider and Terra only for the explicit experiment configuration", () => {
const previousExperimentProvider = process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER;
const previousModel = process.env.OLLAMA_MODEL;
const previousOpenAIKey = process.env.OPENAI_API_KEY;
process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER = "openai";
process.env.OPENAI_API_KEY = "test-key";
process.env.OLLAMA_MODEL = "qwen-default";
try {
expect(getProvider().constructor.name).toBe("OpenAIReconstructionProvider");
expect(getProviderModelName()).toBe("gpt-5.6-terra");
} finally {
if (previousExperimentProvider === undefined) delete process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER;
else process.env.CONFIDENCE_ENGINE_EXPERIMENT_PROVIDER = previousExperimentProvider;
if (previousModel === undefined) delete process.env.OLLAMA_MODEL;
else process.env.OLLAMA_MODEL = previousModel;
if (previousOpenAIKey === undefined) delete process.env.OPENAI_API_KEY;
else process.env.OPENAI_API_KEY = previousOpenAIKey;
}
});
it("uses the configured model for the chat probe and keeps the chat path", async () => { it("uses the configured model for the chat probe and keeps the chat path", async () => {
const originalBaseUrl = process.env.OLLAMA_BASE_URL; const originalBaseUrl = process.env.OLLAMA_BASE_URL;
const fetchSpy = vi.fn() const fetchSpy = vi.fn()