confidence-engine/scripts/reproduce-multi-turn-investigation.mjs

const BASE_URL =
  process.env.CONFIDENCE_ENGINE_BASE_URL || "http://127.0.0.1:3000";

// ── Bounded execution configuration ──────────────────────
// Every update call is explicit and bounded — no implicit retries.
const config = {
  scenario: "Should I relocate my engineering team from London to Manchester?",
  maxUpdates: 2,              // strict upper bound on update calls
  answers: [                   // positional; answer[i] used for Update i+1
    "We're looking at this mainly for cost reduction — roughly £2M annual savings on office overhead.",
    "We don't want to increase staff turnover or lose key engineers as part of the move.",
  ],
};

// ── Call accounting (reflects actual API calls, not successes) ──
const calls = { startCalls: 0, updateCalls: 0 };

async function postJson(path, body) {
  const response = await fetch(`${BASE_URL}${path}`, {
    method: "POST",
    headers: { "content-type": "application/json" },
    body: JSON.stringify(body),
  });
  const json = await response.json();
  return { status: response.status, json };
}

function nodeCount(g) {
  return g?.nodes?.length ?? 0;
}

function edgeCount(g) {
  return g?.edges?.length ?? 0;
}

async function main() {
  // ── Start (exactly one call, no retry) ────────────────
  const startResult = await postJson("/api/cases/start", { scenario: config.scenario });
  calls.startCalls++;

  if (!startResult.json?.success) {
    console.log("=== START ===");
    console.log(`HTTP status: ${startResult.status}`);
    console.log(`errors: ${JSON.stringify(startResult.json?.errors ?? startResult.json?.message ?? null)}`);
    reportCallAccounting();
    process.exitCode = 1;
    return;
  }

  const situationGraph = startResult.json.situationGraph;
  let selectedQuestion = startResult.json.selectedQuestion?.question ?? null;

  console.log("=== START ===");
  console.log(`HTTP status: ${startResult.status}`);
  console.log(`stage: ${startResult.json.stage ?? "unknown"}`);
  console.log(`selected question: ${JSON.stringify(selectedQuestion)}`);
  console.log(`node count: ${nodeCount(situationGraph)}`);
  console.log(`edge count: ${edgeCount(situationGraph)}`);

  if (!selectedQuestion) {
    console.log("ERROR: No selected question returned from start. Stopping.");
    process.exitCode = 1;
    return;
  }

  // ── Update loop (bounded, explicit, no retry) ──────────
  for (let i = 0; i < Math.min(config.maxUpdates, config.answers.length); i++) {
    const updateNum = i + 1;
    const prevQ = selectedQuestion;
    let updateResult = await postJson("/api/cases/update", {
      situationGraph,
      previousQuestion: prevQ,
      answer: config.answers[i],
    });
    calls.updateCalls++;

    if (!updateResult.json?.success) {
      console.log(`\n=== UPDATE ${updateNum} ===`);
      console.log(`HTTP status: ${updateResult.status}`);
      console.log(`stage: ${updateResult.json?.stage ?? "unknown"}`);
      console.log(`selected question: null`);
      console.log(`node count: ${nodeCount(situationGraph)}`);
      console.log(`edge count: ${edgeCount(situationGraph)}`);
      console.log(`error/validation summary: ${JSON.stringify(updateResult.json?.errors ?? updateResult.json?.proposalErrors ?? updateResult.json?.graphValidationErrors ?? updateResult.json?.validationErrors ?? updateResult.json?.message ?? null)}`);

      // Preserve rejectedProposalSnapshot diagnostics if present
      if (updateResult.json?.diagnostics?.rejectedProposalSnapshot) {
        console.log(`\ndiagnostics.rejectedProposalSnapshot: ${JSON.stringify(updateResult.json.diagnostics.rejectedProposalSnapshot, null, 2)}`);
      }

      // ── Capture structuralActionRequired from rejected snapshot if present ──
      const rejectedSnapshot = updateResult.json?.diagnostics?.rejectedProposalSnapshot ?? null;
      if (rejectedSnapshot && "structuralActionRequired" in rejectedSnapshot) {
        console.log(`structuralActionRequired (from rejected proposal snapshot): ${JSON.stringify(rejectedSnapshot.structuralActionRequired)}`);
      } else {
        console.log(`structuralActionRequired: UNAVAILABLE`);
      }

      reportCallAccounting();
      console.log("\n*** UPDATE REJECTED — STOPPING (no retry). ***");
      process.exitCode = 1;
      return;  // rejection stops the chain immediately
    }

    const updatedGraph = updateResult.json.updatedSituationGraph;
    selectedQuestion = updateResult.json.selectedQuestion?.question ?? null;

    console.log(`\n=== UPDATE ${updateNum} ===`);
    console.log(`HTTP status: ${updateResult.status}`);
    console.log(`stage: ${updateResult.json.stage ?? "unknown"}`);
    console.log(`proposal/apply success: ${updateResult.json.proposal?.success ?? updateResult.json.applySuccess ?? null}`);

    // ── Capture accepted answer meaning ─────────────────────
    const am = updateResult.json.answerMeaning ?? null;
    if (am) {
      console.log(`answerMeaning.userSupportedMeaning: ${JSON.stringify(am.userSupportedMeaning ?? null)}`);
      console.log(`answerMeaning.possibleInference: ${JSON.stringify(am.possibleInference ?? null)}`);
      console.log(`answerMeaning.supportCategory: ${JSON.stringify(am.supportCategory ?? null)}`);
      console.log(`answerMeaning.resolutionGuidance: ${JSON.stringify(am.resolutionGuidance ?? null)}`);
    }

    // ── Capture accepted structural mutation fields ────────
    const proposal = updateResult.json.updatedProposal ?? updateResult.json.proposal ?? null;
    if (proposal) {
      console.log(`updatedNodes: ${JSON.stringify(proposal.updatedNodes ?? [])}`);
      console.log(`resolvedUnknownNodeIds: ${JSON.stringify(proposal.resolvedUnknownNodeIds ?? [])}`);
      console.log(`addedNodes: ${JSON.stringify(proposal.addedNodes ?? [])}`);
      console.log(`addedEdges: ${JSON.stringify(proposal.addedEdges ?? [])}`);
    }

    // ── Capture structuralActionRequired from accepted update ─
    const sar = updateResult.json.structuralActionRequired;
    if (sar === undefined || sar === null) {
      console.log(`structuralActionRequired: null`);
    } else {
      console.log(`structuralActionRequired: ${JSON.stringify(sar)}`);
    }

    // ── Capture selectedQuestion node reference ────────────
    const sq = updateResult.json.selectedQuestion ?? null;
    if (sq && typeof sq === "object") {
      console.log(`selectedQuestion: ${JSON.stringify(sq.question ?? null)}`);
      if (sq.nodeId) {
        console.log(`selectedQuestion.nodeId: ${JSON.stringify(sq.nodeId)}`);
      }
    }

    // ── Compact structural snapshot of resulting graph ─────
    const nodes = updatedGraph?.nodes ?? [];
    const edges = updatedGraph?.edges ?? [];
    console.log(`\nresulting graph (${nodes.length} nodes, ${edges.length} edges):`);
    for (const n of nodes) {
      console.log(`  node: id=${n.id ?? n.nodeId}, kind=${n.kind}, label=${n.label ?? n.description ?? ""}, status=${n.status}`);
    }
    for (const e of edges) {
      console.log(`  edge: from=${e.fromNodeId ?? e.from}, to=${e.toNodeId ?? e.to}, relationship=${e.relationship}`);
    }

    situationGraph = updatedGraph;
  }
}

function reportCallAccounting() {
  const totalCalls = calls.startCalls + calls.updateCalls;
  console.log("\n=== CALL ACCOUNTING ===");
  console.log(`startCalls: ${calls.startCalls}`);
  console.log(`updateCalls: ${calls.updateCalls}`);
  console.log(`totalCalls: ${totalCalls}`);
}

main().catch((error) => {
  console.error(error instanceof Error ? error.message : String(error));
  reportCallAccounting();
  process.exitCode = 1;
});