confidence-engine/scripts/reproduce-multi-turn-investigation.mjs

import fs from "fs";
import { fileURLToPath } from "url";
import path from "path";

// Default fixture — used when FIXTURE_PATH is not set (existing behaviour).
const DEFAULT_FIXTURE_PATH = path.resolve(
  path.dirname(fileURLToPath(import.meta.url)),
  "../tests/fixtures/pre-anchored-update-savings-realism.json",
);

/**
 * Resolves the fixture path for updateOnly mode.
 * FIXTURE_PATH (env) overrides the default if set and non-empty.
 */
function resolveFixturePath() {
  const envPath = process.env.FIXTURE_PATH;
  if (envPath && String(envPath).trim() !== "") {
    return path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", envPath);
  }
  return DEFAULT_FIXTURE_PATH;
}

const BASE_URL =
  process.env.CONFIDENCE_ENGINE_BASE_URL || "http://127.0.0.1:3000";

// ── Bounded execution configuration ──────────────────────
// Every update call is explicit and bounded — no implicit retries.
const config = {
  scenario: "Should I relocate my engineering team from London to Manchester?",
  maxUpdates: 2,              // strict upper bound on update calls
  answers: [                   // positional; answer[i] used for Update i+1
    "We're looking at this mainly for cost reduction — roughly £2M annual savings on office overhead.",
    "We don't want to increase staff turnover or lose key engineers as part of the move.",
  ],
};

// ── Mode selector ────────────────────────────────────────
const fixtureMode = process.env.FIXTURE_MODE;

// ── Call accounting (reflects actual API calls, not successes) ──
const calls = { startCalls: 0, updateCalls: 0 };

async function postJson(path, body) {
  const response = await fetch(`${BASE_URL}${path}`, {
    method: "POST",
    headers: { "content-type": "application/json" },
    body: JSON.stringify(body),
  });
  const json = await response.json();
  return { status: response.status, json };
}

function nodeCount(g) {
  return g?.nodes?.length ?? 0;
}

function edgeCount(g) {
  return g?.edges?.length ?? 0;
}

async function main() {
  // ── Mode dispatch ──────────────────────────────────────
  if (fixtureMode === "updateOnly") {
    await runUpdateOnlyMode();
    reportCallAccounting();
    return;
  }

  if (fixtureMode !== undefined) {
    console.error(`ERROR: Unsupported FIXTURE_MODE="${fixtureMode}". Use "updateOnly" or unset.`);
    process.exitCode = 1;
    return;
  }

  // ── Normal mode: Start→Update chain (unchanged from original) ──
  const startResult = await postJson("/api/cases/start", { scenario: config.scenario });
  calls.startCalls++;

  if (!startResult.json?.success) {
    console.log("=== START ===");
    console.log(`HTTP status: ${startResult.status}`);
    console.log(`errors: ${JSON.stringify(startResult.json?.errors ?? startResult.json?.message ?? null)}`);
    reportCallAccounting();
    process.exitCode = 1;
    return;
  }

  const situationGraph = startResult.json.situationGraph;
  let selectedQuestion = startResult.json.selectedQuestion?.question ?? null;

  console.log("=== START ===");
  console.log(`HTTP status: ${startResult.status}`);
  console.log(`stage: ${startResult.json.stage ?? "unknown"}`);
  console.log(`selected question: ${JSON.stringify(selectedQuestion)}`);
  console.log(`node count: ${nodeCount(situationGraph)}`);
  console.log(`edge count: ${edgeCount(situationGraph)}`);

  if (!selectedQuestion) {
    console.log("ERROR: No selected question returned from start. Stopping.");
    process.exitCode = 1;
    return;
  }

  // ── Update loop (bounded, explicit, no retry) ──────────
  for (let i = 0; i < Math.min(config.maxUpdates, config.answers.length); i++) {
    const updateNum = i + 1;
    const prevQ = selectedQuestion;
    let updateResult = await postJson("/api/cases/update", {
      situationGraph,
      previousQuestion: prevQ,
      answer: config.answers[i],
    });
    calls.updateCalls++;

    if (!updateResult.json?.success) {
      console.log(`\n=== UPDATE ${updateNum} ===`);
      console.log(`HTTP status: ${updateResult.status}`);
      console.log(`stage: ${updateResult.json?.stage ?? "unknown"}`);
      console.log(`selected question: null`);
      console.log(`node count: ${nodeCount(situationGraph)}`);
      console.log(`edge count: ${edgeCount(situationGraph)}`);
      console.log(`error/validation summary: ${JSON.stringify(updateResult.json?.errors ?? updateResult.json?.proposalErrors ?? updateResult.json?.graphValidationErrors ?? updateResult.json?.validationErrors ?? updateResult.json?.message ?? null)}`);

      // Preserve rejectedProposalSnapshot diagnostics if present
      if (updateResult.json?.diagnostics?.rejectedProposalSnapshot) {
        console.log(`\ndiagnostics.rejectedProposalSnapshot: ${JSON.stringify(updateResult.json.diagnostics.rejectedProposalSnapshot, null, 2)}`);
      }

      // ── Capture structuralActionRequired from rejected snapshot if present ──
      const rejectedSnapshot = updateResult.json?.diagnostics?.rejectedProposalSnapshot ?? null;
      if (rejectedSnapshot && "structuralActionRequired" in rejectedSnapshot) {
        console.log(`structuralActionRequired (from rejected proposal snapshot): ${JSON.stringify(rejectedSnapshot.structuralActionRequired)}`);
      } else {
        console.log(`structuralActionRequired: UNAVAILABLE`);
      }

      reportCallAccounting();
      console.log("\n*** UPDATE REJECTED — STOPPING (no retry). ***");
      process.exitCode = 1;
      return;  // rejection stops the chain immediately
    }

    const updatedGraph = updateResult.json.updatedSituationGraph;
    selectedQuestion = updateResult.json.selectedQuestion?.question ?? null;

    console.log(`\n=== UPDATE ${updateNum} ===`);
    console.log(`HTTP status: ${updateResult.status}`);
    console.log(`stage: ${updateResult.json.stage ?? "unknown"}`);
    console.log(`proposal/apply success: ${updateResult.json.proposal?.success ?? updateResult.json.applySuccess ?? null}`);

    // ── Capture accepted structural mutation fields ────────
    const proposal = updateResult.json.updatedProposal ?? updateResult.json.proposal ?? null;

    // ── Capture answerMeaning and structuralActionRequired from proposal (production path) ─
    const am = proposal?.answerMeaning ?? null;
    if (am) {
      console.log(`answerMeaning.userSupportedMeaning: ${JSON.stringify(am.userSupportedMeaning ?? null)}`);
      console.log(`answerMeaning.possibleInference: ${JSON.stringify(am.possibleInference ?? null)}`);
      console.log(`answerMeaning.supportCategory: ${JSON.stringify(am.supportCategory ?? null)}`);
      console.log(`answerMeaning.resolutionGuidance: ${JSON.stringify(am.resolutionGuidance ?? null)}`);
    }

    if (proposal) {
      console.log(`updatedNodes: ${JSON.stringify(proposal.updatedNodes ?? [])}`);
      console.log(`resolvedUnknownNodeIds: ${JSON.stringify(proposal.resolvedUnknownNodeIds ?? [])}`);
      console.log(`addedNodes: ${JSON.stringify(proposal.addedNodes ?? [])}`);
      console.log(`addedEdges: ${JSON.stringify(proposal.addedEdges ?? [])}`);
    }

    // ── Capture structuralActionRequired from accepted update ─
    const sar = proposal?.structuralActionRequired;
    if (sar === undefined || sar === null) {
      console.log(`structuralActionRequired: null`);
    } else {
      console.log(`structuralActionRequired: ${JSON.stringify(sar)}`);
    }

    // ── Capture selectedQuestion node reference ────────────
    const sq = updateResult.json.selectedQuestion ?? null;
    if (sq && typeof sq === "object") {
      console.log(`selectedQuestion: ${JSON.stringify(sq.question ?? null)}`);
      if (sq.nodeId) {
        console.log(`selectedQuestion.nodeId: ${JSON.stringify(sq.nodeId)}`);
      }
    }

    // ── Compact structural snapshot of resulting graph ─────
    const nodes = updatedGraph?.nodes ?? [];
    const edges = updatedGraph?.edges ?? [];
    console.log(`\nresulting graph (${nodes.length} nodes, ${edges.length} edges):`);
    for (const n of nodes) {
      console.log(`  node: id=${n.id ?? n.nodeId}, kind=${n.kind}, label=${n.label ?? n.description ?? ""}, status=${n.status}`);
    }
    for (const e of edges) {
      console.log(`  edge: from=${e.fromNodeId ?? e.from}, to=${e.toNodeId ?? e.to}, relationship=${e.relationship}`);
    }

    situationGraph = updatedGraph;
  }
}

// ── Pre-anchored update-only mode (57J.78) ───────────────

/**
 * Loads the committed pre-anchored fixture, skips Start,
 * sends exactly one Update through the production HTTP route,
 * and preserves all hardened capture/no-retry behaviour.
 */
async function runUpdateOnlyMode() {
  const answer = process.env.ANSWER_2;

  if (!answer || String(answer).trim() === "") {
    console.log("BLOCKED - missing ANSWER_2");
    return; // zero live calls made
  }

  const fixturePath = resolveFixturePath();

  // Load committed fixture — single source of truth.
  let fixtureData;
  try {
    const raw = fs.readFileSync(fixturePath, "utf-8");
    fixtureData = JSON.parse(raw);
  } catch (err) {
    console.error(`ERROR: Cannot load pre-anchored fixture from ${fixturePath}`);
    process.exitCode = 1;
    return;
  }

  const initialGraph = fixtureData.graph;

  // Verify fixture integrity before proceeding.
  const nodes = initialGraph.nodes;
  const edges = initialGraph.edges;

  // Generic anchor check: any unresolved unknown node (supports both savings-realism and decision-options fixtures).
  const unresolvedNodes = nodes.filter(
    (n) => n.kind === "unknown" && n.status === "unknown",
  );

  if (unresolvedNodes.length < 1) {
    console.log(`ERROR: pre-anchored fixture must contain at least one unresolved unknown node (found ${unresolvedNodes.length}).`);
    process.exitCode = 1;
    return;
  }

  // Use the first unresolved unknown as the anchor for previousQuestion derivation.
  const anchorNode = unresolvedNodes[0];

  // Pre-call compact fixture snapshot.
  console.log(`\n=== UPDATE-ONLY MODE ===`);
  console.log(`fixture node count: ${nodes.length}`);
  console.log(`fixture edge count: ${edges.length}`);
  console.log(`anchor node:`);
  console.log(`  id: ${anchorNode.id}`);
  console.log(`  label: ${anchorNode.label}`);
  console.log(`  kind: ${anchorNode.kind}`);
  console.log(`  status: ${anchorNode.status}`);

  // Deep-copy so mutation doesn't corrupt the original fixture.
  const situationGraph = JSON.parse(JSON.stringify(initialGraph));

  // Derive previousQuestion from the fixture's unresolved_question or anchor node label.
  let selectedQuestion = fixtureData.unresolved_question ?? anchorNode.label;

  // ── Exactly one Update through production route ────────
  const updateNum = 1;
  let updateResult = await postJson("/api/cases/update", {
    situationGraph,
    previousQuestion: selectedQuestion,
    answer: String(answer),
  });
  calls.updateCalls++;

  if (!updateResult.json?.success) {
    console.log(`\n=== UPDATE ${updateNum} ===`);
    console.log(`HTTP status: ${updateResult.status}`);
    console.log(`stage: ${updateResult.json?.stage ?? "unknown"}`);
    console.log(`selected question: null`);
    console.log(`node count: ${nodeCount(situationGraph)}`);
    console.log(`edge count: ${edgeCount(situationGraph)}`);
    console.log(
      `error/validation summary: ${JSON.stringify(updateResult.json?.errors ?? updateResult.json?.proposalErrors ?? updateResult.json?.graphValidationErrors ?? updateResult.json?.validationErrors ?? updateResult.json?.message ?? null)}`,
    );

    // Preserve rejectedProposalSnapshot diagnostics if present.
    if (updateResult.json?.diagnostics?.rejectedProposalSnapshot) {
      console.log(
        `\ndiagnostics.rejectedProposalSnapshot: ${JSON.stringify(updateResult.json.diagnostics.rejectedProposalSnapshot, null, 2)}`,
      );
    }

    // Capture structuralActionRequired from rejected snapshot.
    const rejectedSnapshot = updateResult.json?.diagnostics?.rejectedProposalSnapshot ?? null;
    if (rejectedSnapshot && "structuralActionRequired" in rejectedSnapshot) {
      console.log(
        `structuralActionRequired (from rejected proposal snapshot): ${JSON.stringify(rejectedSnapshot.structuralActionRequired)}`,
      );
    } else {
      console.log(`structuralActionRequired: UNAVAILABLE`);
    }

    console.log("\n*** UPDATE REJECTED — STOPPING (no retry). ***");
    process.exitCode = 1;
    return;
  }

  const updatedGraph = updateResult.json.updatedSituationGraph;
  selectedQuestion = updateResult.json.selectedQuestion?.question ?? null;

  // ── Capture accepted answer meaning ─────────────────────
  const am = updateResult.json.answerMeaning ?? null;
  if (am) {
    console.log(`answerMeaning.userSupportedMeaning: ${JSON.stringify(am.userSupportedMeaning ?? null)}`);
    console.log(`answerMeaning.possibleInference: ${JSON.stringify(am.possibleInference ?? null)}`);
    console.log(`answerMeaning.supportCategory: ${JSON.stringify(am.supportCategory ?? null)}`);
    console.log(`answerMeaning.resolutionGuidance: ${JSON.stringify(am.resolutionGuidance ?? null)}`);
  }

  // ── Capture accepted structural mutation fields ────────
  const proposal = updateResult.json.updatedProposal ?? updateResult.json.proposal ?? null;
  if (proposal) {
    console.log(`updatedNodes: ${JSON.stringify(proposal.updatedNodes ?? [])}`);
    console.log(`resolvedUnknownNodeIds: ${JSON.stringify(proposal.resolvedUnknownNodeIds ?? [])}`);
    console.log(`addedNodes: ${JSON.stringify(proposal.addedNodes ?? [])}`);
    console.log(`addedEdges: ${JSON.stringify(proposal.addedEdges ?? [])}`);
  }

  // ── Capture structuralActionRequired from accepted update ─
  const sar = updateResult.json.structuralActionRequired;
  if (sar === undefined || sar === null) {
    console.log(`structuralActionRequired: null`);
  } else {
    console.log(`structuralActionRequired: ${JSON.stringify(sar)}`);
  }

  // ── Capture selectedQuestion node reference ────────────
  const sq = updateResult.json.selectedQuestion ?? null;
  if (sq && typeof sq === "object") {
    console.log(`selectedQuestion: ${JSON.stringify(sq.question ?? null)}`);
    if (sq.nodeId) {
      console.log(`selectedQuestion.nodeId: ${JSON.stringify(sq.nodeId)}`);
    }
  }

  // ── Capture persistent graph after update ───────────────
  const pNodes = updatedGraph?.nodes ?? [];
  const pEdges = updatedGraph?.edges ?? [];
  console.log(`\nresulting persistent graph (${pNodes.length} nodes, ${pEdges.length} edges):`);
  for (const n of pNodes) {
    console.log(`  node: id=${n.id ?? n.nodeId}, kind=${n.kind}, label=${n.label ?? n.description ?? ""}, status=${n.status}`);
  }
  for (const e of pEdges) {
    console.log(`  edge: from=${e.fromNodeId ?? e.from}, to=${e.toNodeId ?? e.to}, relationship=${e.relationship}`);
  }

  // ── UPDATE-ONLY EXIT — no retry, no additional calls ───
}

function reportCallAccounting() {
  const totalCalls = calls.startCalls + calls.updateCalls;
  console.log("\n=== CALL ACCOUNTING ===");
  console.log(`startCalls: ${calls.startCalls}`);
  console.log(`updateCalls: ${calls.updateCalls}`);
  console.log(`totalCalls: ${totalCalls}`);
}

main().catch((error) => {
  console.error(error instanceof Error ? error.message : String(error));
  reportCallAccounting();
  process.exitCode = 1;
});