import { describe, it, expect } from "vitest"; // ── Deterministic harness tests (no Ollama, no dev-server) ── // These verify the canonical harness logic via a synchronous simulator // that mirrors exactly what reproduce-multi-turn-investigation.mjs does. describe("reproduce-multi-turn-investigation harness: one-shot semantics", () => { /** * Synchronous simulator of the harness — mirrors every branching path. */ function runSimulation(cfg) { let startCalls = 0; let updateCalls = 0; let apiLog = []; // Mock API (mirrors expected production contract) const api = { post(path, body) { if (path === "/api/cases/start") { apiLog.push({ step: "start" }); const failScenario = body.scenario == null || String(body.scenario).includes("_fail"); return { status: failScenario ? 500 : 200, json: () => failScenario ? { success: false, errors: ["start failed"] } : { success: true, situationGraph: { nodes: [], edges: [] }, selectedQuestion: { question: "q" } }, }; } if (path === "/api/cases/update") { apiLog.push({ step: "update", answer: body.answer }); const shouldReject = typeof body.answer === "string" && body.answer.includes("_reject"); return { status: shouldReject ? 422 : 200, json: () => shouldReject ? { success: false, stage: "proposal_compatibility", errors: ["proposal_compatibility rejection"], diagnostics: { rejectedProposalSnapshot: { userSupportedMeaning: "rejected", addedNodes: [], addedEdges: [] } } } : { success: true, stage: "update_applied", updatedSituationGraph: { nodes: [], edges: [] }, selectedQuestion: { question: "q2" } }, }; } apiLog.push({ step: "unknown", path }); return { status: 404, json: () => ({ error: "not found" }) }; }, }; // --- START (exactly one call, no retry) --- const startResp = api.post("/api/cases/start", { scenario: cfg.scenario || "test" }); startCalls = 1; const sj = startResp.json(); if (!sj.success) { return { startCalls, updateCalls, type: "start_failure", exitCode: 1, apiLog }; } let graph = sj.situationGraph; let question = sj.selectedQuestion ? sj.selectedQuestion.question : null; if (question == null) { return { startCalls, updateCalls, type: "no_question", exitCode: 1, apiLog }; } // --- UPDATES (bounded loop, no retry) --- const answers = cfg.answers || []; const maxUpdates = typeof cfg.maxUpdates === "number" ? cfg.maxUpdates : 2; const limit = Math.min(maxUpdates, answers.length); for (let i = 0; i < limit; i++) { const upResp = api.post("/api/cases/update", { situationGraph: graph, previousQuestion: question, answer: answers[i], }); updateCalls++; const uj = upResp.json(); if (!uj.success) { return { startCalls, updateCalls, type: "update_rejection", exitCode: 1, updateNum: i + 1, apiLog }; } graph = uj.updatedSituationGraph; question = uj.selectedQuestion ? uj.selectedQuestion.question : null; } return { startCalls, updateCalls, type: "all_success", exitCode: 0, apiLog }; } // ── Test cases ────────────────────────────────────────── it("Start success → exactly 1 Start call", () => { const r = runSimulation({ scenario: "test_valid_scenario", maxUpdates: 2, answers: ["a1"] }); expect(r.startCalls).toBe(1); expect(r.updateCalls).toBeGreaterThanOrEqual(0); expect(r.type).not.toBe("start_failure"); }); it("Start failure → exactly 1 Start call, no retry", () => { const r = runSimulation({ scenario: "test_fail", maxUpdates: 2, answers: ["a"] }); expect(r.startCalls).toBe(1); expect(r.updateCalls).toBe(0); expect(r.type).toBe("start_failure"); expect(r.exitCode).toBe(1); }); it("Update success → exactly 1 Update call", () => { const r = runSimulation({ scenario: "test_ok", maxUpdates: 1, answers: ["good answer"] }); expect(r.startCalls).toBe(1); expect(r.updateCalls).toBe(1); expect(r.type).toBe("all_success"); expect(r.exitCode).toBe(0); expect(r.apiLog.filter((e) => e.step === "update").length).toBe(1); }); it("proposal_compatibility rejection → exactly 1 Update call, rejection returned unchanged", () => { const r = runSimulation({ scenario: "test_ok", maxUpdates: 1, answers: ["answer_reject"] }); expect(r.startCalls).toBe(1); expect(r.updateCalls).toBe(1); expect(r.type).toBe("update_rejection"); expect(r.exitCode).toBe(1); expect(r.apiLog.filter((e) => e.step === "update").length).toBe(1); }); it("Update 1 rejection → Update 2 is never called", () => { const r = runSimulation({ scenario: "test_ok", maxUpdates: 2, answers: ["answer_reject", "second answer"] }); expect(r.startCalls).toBe(1); expect(r.updateCalls).toBe(1); // only Update 1 was made — rejection stops the chain expect(r.type).toBe("update_rejection"); expect(r.apiLog.filter((e) => e.step === "update").length).toBe(1); }); it("Update 1 success → Update 2 called exactly once when explicitly requested", () => { const r = runSimulation({ scenario: "test_ok", maxUpdates: 2, answers: ["good answer 1", "good answer 2"] }); expect(r.startCalls).toBe(1); expect(r.updateCalls).toBe(2); expect(r.type).toBe("all_success"); expect(r.exitCode).toBe(0); expect(r.apiLog.filter((e) => e.step === "update").length).toBe(2); }); it("Call counters equal actual mocked API invocations", () => { const r = runSimulation({ scenario: "test_ok", maxUpdates: 2, answers: ["good answer 1", "good answer 2"] }); const totalApiCalls = r.apiLog.length; expect(r.startCalls + r.updateCalls).toBe(totalApiCalls); expect(r.startCalls).toBe(1); expect(r.updateCalls).toBe(2); expect(r.apiLog.filter((e) => e.step === "start").length).toBe(1); expect(r.apiLog.filter((e) => e.step === "update").length).toBe(2); }); it("No semantic retry occurs after HTTP 422/valid rejection response", () => { const r = runSimulation({ scenario: "test_ok", maxUpdates: 3, answers: ["answer_reject", "should_not_fire", "also_should_not_fire"] }); expect(r.updateCalls).toBe(1); // Update 1 rejects; no retry. expect(r.type).toBe("update_rejection"); const updateEntries = r.apiLog.filter((e) => e.step === "update"); expect(updateEntries.length).toBe(1); expect(updateEntries[0].answer).toContain("answer_reject"); }); });