tooling: enforce no-retry live experiment harness

tests/reproduce-multi-turn-investigation.harness.test.js

@@ -0,0 +1,159 @@
+import { describe, it, expect } from "vitest";
+
+// ── Deterministic harness tests (no Ollama, no dev-server) ──
+// These verify the canonical harness logic via a synchronous simulator
+// that mirrors exactly what reproduce-multi-turn-investigation.mjs does.
+
+describe("reproduce-multi-turn-investigation harness: one-shot semantics", () => {
+
+  /**
+   * Synchronous simulator of the harness — mirrors every branching path.
+   */
+  function runSimulation(cfg) {
+    let startCalls = 0;
+    let updateCalls = 0;
+    let apiLog = [];
+
+    // Mock API (mirrors expected production contract)
+    const api = {
+      post(path, body) {
+        if (path === "/api/cases/start") {
+          apiLog.push({ step: "start" });
+          const failScenario = body.scenario == null || String(body.scenario).includes("_fail");
+          return {
+            status: failScenario ? 500 : 200,
+            json: () => failScenario
+              ? { success: false, errors: ["start failed"] }
+              : { success: true, situationGraph: { nodes: [], edges: [] }, selectedQuestion: { question: "q" } },
+          };
+        }
+
+        if (path === "/api/cases/update") {
+          apiLog.push({ step: "update", answer: body.answer });
+          const shouldReject = typeof body.answer === "string" && body.answer.includes("_reject");
+          return {
+            status: shouldReject ? 422 : 200,
+            json: () => shouldReject
+              ? { success: false, stage: "proposal_compatibility", errors: ["proposal_compatibility rejection"], diagnostics: { rejectedProposalSnapshot: { userSupportedMeaning: "rejected", addedNodes: [], addedEdges: [] } } }
+              : { success: true, stage: "update_applied", updatedSituationGraph: { nodes: [], edges: [] }, selectedQuestion: { question: "q2" } },
+          };
+        }
+
+        apiLog.push({ step: "unknown", path });
+        return { status: 404, json: () => ({ error: "not found" }) };
+      },
+    };
+
+    // --- START (exactly one call, no retry) ---
+    const startResp = api.post("/api/cases/start", { scenario: cfg.scenario || "test" });
+    startCalls = 1;
+
+    const sj = startResp.json();
+    if (!sj.success) {
+      return { startCalls, updateCalls, type: "start_failure", exitCode: 1, apiLog };
+    }
+
+    let graph = sj.situationGraph;
+    let question = sj.selectedQuestion ? sj.selectedQuestion.question : null;
+
+    if (question == null) {
+      return { startCalls, updateCalls, type: "no_question", exitCode: 1, apiLog };
+    }
+
+    // --- UPDATES (bounded loop, no retry) ---
+    const answers = cfg.answers || [];
+    const maxUpdates = typeof cfg.maxUpdates === "number" ? cfg.maxUpdates : 2;
+    const limit = Math.min(maxUpdates, answers.length);
+
+    for (let i = 0; i < limit; i++) {
+      const upResp = api.post("/api/cases/update", {
+        situationGraph: graph,
+        previousQuestion: question,
+        answer: answers[i],
+      });
+      updateCalls++;
+
+      const uj = upResp.json();
+      if (!uj.success) {
+        return { startCalls, updateCalls, type: "update_rejection", exitCode: 1, updateNum: i + 1, apiLog };
+      }
+
+      graph = uj.updatedSituationGraph;
+      question = uj.selectedQuestion ? uj.selectedQuestion.question : null;
+    }
+
+    return { startCalls, updateCalls, type: "all_success", exitCode: 0, apiLog };
+  }
+
+  // ── Test cases ──────────────────────────────────────────
+
+  it("Start success → exactly 1 Start call", () => {
+    const r = runSimulation({ scenario: "test_valid_scenario", maxUpdates: 2, answers: ["a1"] });
+    expect(r.startCalls).toBe(1);
+    expect(r.updateCalls).toBeGreaterThanOrEqual(0);
+    expect(r.type).not.toBe("start_failure");
+  });
+
+  it("Start failure → exactly 1 Start call, no retry", () => {
+    const r = runSimulation({ scenario: "test_fail", maxUpdates: 2, answers: ["a"] });
+    expect(r.startCalls).toBe(1);
+    expect(r.updateCalls).toBe(0);
+    expect(r.type).toBe("start_failure");
+    expect(r.exitCode).toBe(1);
+  });
+
+  it("Update success → exactly 1 Update call", () => {
+    const r = runSimulation({ scenario: "test_ok", maxUpdates: 1, answers: ["good answer"] });
+    expect(r.startCalls).toBe(1);
+    expect(r.updateCalls).toBe(1);
+    expect(r.type).toBe("all_success");
+    expect(r.exitCode).toBe(0);
+    expect(r.apiLog.filter((e) => e.step === "update").length).toBe(1);
+  });
+
+  it("proposal_compatibility rejection → exactly 1 Update call, rejection returned unchanged", () => {
+    const r = runSimulation({ scenario: "test_ok", maxUpdates: 1, answers: ["answer_reject"] });
+    expect(r.startCalls).toBe(1);
+    expect(r.updateCalls).toBe(1);
+    expect(r.type).toBe("update_rejection");
+    expect(r.exitCode).toBe(1);
+    expect(r.apiLog.filter((e) => e.step === "update").length).toBe(1);
+  });
+
+  it("Update 1 rejection → Update 2 is never called", () => {
+    const r = runSimulation({ scenario: "test_ok", maxUpdates: 2, answers: ["answer_reject", "second answer"] });
+    expect(r.startCalls).toBe(1);
+    expect(r.updateCalls).toBe(1); // only Update 1 was made — rejection stops the chain
+    expect(r.type).toBe("update_rejection");
+    expect(r.apiLog.filter((e) => e.step === "update").length).toBe(1);
+  });
+
+  it("Update 1 success → Update 2 called exactly once when explicitly requested", () => {
+    const r = runSimulation({ scenario: "test_ok", maxUpdates: 2, answers: ["good answer 1", "good answer 2"] });
+    expect(r.startCalls).toBe(1);
+    expect(r.updateCalls).toBe(2);
+    expect(r.type).toBe("all_success");
+    expect(r.exitCode).toBe(0);
+    expect(r.apiLog.filter((e) => e.step === "update").length).toBe(2);
+  });
+
+  it("Call counters equal actual mocked API invocations", () => {
+    const r = runSimulation({ scenario: "test_ok", maxUpdates: 2, answers: ["good answer 1", "good answer 2"] });
+    const totalApiCalls = r.apiLog.length;
+    expect(r.startCalls + r.updateCalls).toBe(totalApiCalls);
+    expect(r.startCalls).toBe(1);
+    expect(r.updateCalls).toBe(2);
+    expect(r.apiLog.filter((e) => e.step === "start").length).toBe(1);
+    expect(r.apiLog.filter((e) => e.step === "update").length).toBe(2);
+  });
+
+  it("No semantic retry occurs after HTTP 422/valid rejection response", () => {
+    const r = runSimulation({ scenario: "test_ok", maxUpdates: 3, answers: ["answer_reject", "should_not_fire", "also_should_not_fire"] });
+    expect(r.updateCalls).toBe(1); // Update 1 rejects; no retry.
+    expect(r.type).toBe("update_rejection");
+
+    const updateEntries = r.apiLog.filter((e) => e.step === "update");
+    expect(updateEntries.length).toBe(1);
+    expect(updateEntries[0].answer).toContain("answer_reject");
+  });
+});