confidence-engine/scripts/reproduce-multi-turn-investigation.mjs

const BASE_URL =
  process.env.CONFIDENCE_ENGINE_BASE_URL || "http://127.0.0.1:3000";

// ── Bounded execution configuration ──────────────────────
// Every update call is explicit and bounded — no implicit retries.
const config = {
  scenario: "Should I relocate my engineering team from London to Manchester?",
  maxUpdates: 2,              // strict upper bound on update calls
  answers: [                   // positional; answer[i] used for Update i+1
    "We're looking at this mainly for cost reduction — roughly £2M annual savings on office overhead.",
    "We don't want to increase staff turnover or lose key engineers as part of the move.",
  ],
};

// ── Call accounting (reflects actual API calls, not successes) ──
const calls = { startCalls: 0, updateCalls: 0 };

async function postJson(path, body) {
  const response = await fetch(`${BASE_URL}${path}`, {
    method: "POST",
    headers: { "content-type": "application/json" },
    body: JSON.stringify(body),
  });
  const json = await response.json();
  return { status: response.status, json };
}

function nodeCount(g) {
  return g?.nodes?.length ?? 0;
}

function edgeCount(g) {
  return g?.edges?.length ?? 0;
}

async function main() {
  // ── Start (exactly one call, no retry) ────────────────
  const startResult = await postJson("/api/cases/start", { scenario: config.scenario });
  calls.startCalls++;

  if (!startResult.json?.success) {
    console.log("=== START ===");
    console.log(`HTTP status: ${startResult.status}`);
    console.log(`errors: ${JSON.stringify(startResult.json?.errors ?? startResult.json?.message ?? null)}`);
    reportCallAccounting();
    process.exitCode = 1;
    return;
  }

  const situationGraph = startResult.json.situationGraph;
  let selectedQuestion = startResult.json.selectedQuestion?.question ?? null;

  console.log("=== START ===");
  console.log(`HTTP status: ${startResult.status}`);
  console.log(`stage: ${startResult.json.stage ?? "unknown"}`);
  console.log(`selected question: ${JSON.stringify(selectedQuestion)}`);
  console.log(`node count: ${nodeCount(situationGraph)}`);
  console.log(`edge count: ${edgeCount(situationGraph)}`);

  if (!selectedQuestion) {
    console.log("ERROR: No selected question returned from start. Stopping.");
    process.exitCode = 1;
    return;
  }

  // ── Update loop (bounded, explicit, no retry) ──────────
  for (let i = 0; i < Math.min(config.maxUpdates, config.answers.length); i++) {
    const updateNum = i + 1;
    const prevQ = selectedQuestion;
    let updateResult = await postJson("/api/cases/update", {
      situationGraph,
      previousQuestion: prevQ,
      answer: config.answers[i],
    });
    calls.updateCalls++;

    if (!updateResult.json?.success) {
      console.log(`\n=== UPDATE ${updateNum} ===`);
      console.log(`HTTP status: ${updateResult.status}`);
      console.log(`stage: ${updateResult.json?.stage ?? "unknown"}`);
      console.log(`selected question: null`);
      console.log(`node count: ${nodeCount(situationGraph)}`);
      console.log(`edge count: ${edgeCount(situationGraph)}`);
      console.log(`error/validation summary: ${JSON.stringify(updateResult.json?.errors ?? updateResult.json?.proposalErrors ?? updateResult.json?.graphValidationErrors ?? updateResult.json?.validationErrors ?? updateResult.json?.message ?? null)}`);

      // Preserve rejectedProposalSnapshot diagnostics if present
      if (updateResult.json?.diagnostics?.rejectedProposalSnapshot) {
        console.log(`\ndiagnostics.rejectedProposalSnapshot: ${JSON.stringify(updateResult.json.diagnostics.rejectedProposalSnapshot, null, 2)}`);
      }

      reportCallAccounting();
      console.log("\n*** UPDATE REJECTED — STOPPING (no retry). ***");
      process.exitCode = 1;
      return;  // rejection stops the chain immediately
    }

    const updatedGraph = updateResult.json.updatedSituationGraph;
    selectedQuestion = updateResult.json.selectedQuestion?.question ?? null;

    console.log(`\n=== UPDATE ${updateNum} ===`);
    console.log(`HTTP status: ${updateResult.status}`);
    console.log(`stage: ${updateResult.json.stage ?? "unknown"}`);
    console.log(`proposal/apply success: ${updateResult.json.proposal?.success ?? updateResult.json.applySuccess ?? null}`);
    console.log(`selected question: ${JSON.stringify(selectedQuestion)}`);
    console.log(`node count: ${nodeCount(updatedGraph)}`);
    console.log(`edge count: ${edgeCount(updatedGraph)}`);

    situationGraph = updatedGraph;
  }
}

function reportCallAccounting() {
  const totalCalls = calls.startCalls + calls.updateCalls;
  console.log("\n=== CALL ACCOUNTING ===");
  console.log(`startCalls: ${calls.startCalls}`);
  console.log(`updateCalls: ${calls.updateCalls}`);
  console.log(`totalCalls: ${totalCalls}`);
}

main().catch((error) => {
  console.error(error instanceof Error ? error.message : String(error));
  reportCallAccounting();
  process.exitCode = 1;
});