tooling: enforce no-retry live experiment harness
This commit is contained in:
@@ -1,13 +1,19 @@
|
||||
const BASE_URL =
|
||||
process.env.CONFIDENCE_ENGINE_BASE_URL || "http://127.0.0.1:3000";
|
||||
|
||||
const scenario =
|
||||
"Should I relocate my engineering team from London to Manchester?";
|
||||
// ── Bounded execution configuration ──────────────────────
|
||||
// Every update call is explicit and bounded — no implicit retries.
|
||||
const config = {
|
||||
scenario: "Should I relocate my engineering team from London to Manchester?",
|
||||
maxUpdates: 2, // strict upper bound on update calls
|
||||
answers: [ // positional; answer[i] used for Update i+1
|
||||
"We're looking at this mainly for cost reduction — roughly £2M annual savings on office overhead.",
|
||||
"We don't want to increase staff turnover or lose key engineers as part of the move.",
|
||||
],
|
||||
};
|
||||
|
||||
const answers = [
|
||||
"We're looking at this mainly for cost reduction — roughly £2M annual savings on office overhead.",
|
||||
"We don't want to increase staff turnover or lose key engineers as part of the move.",
|
||||
];
|
||||
// ── Call accounting (reflects actual API calls, not successes) ──
|
||||
const calls = { startCalls: 0, updateCalls: 0 };
|
||||
|
||||
async function postJson(path, body) {
|
||||
const response = await fetch(`${BASE_URL}${path}`, {
|
||||
@@ -28,13 +34,15 @@ function edgeCount(g) {
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// ── Start ──────────────────────────────────────────────
|
||||
const startResult = await postJson("/api/cases/start", { scenario });
|
||||
// ── Start (exactly one call, no retry) ────────────────
|
||||
const startResult = await postJson("/api/cases/start", { scenario: config.scenario });
|
||||
calls.startCalls++;
|
||||
|
||||
if (!startResult.json?.success) {
|
||||
console.log("=== START ===");
|
||||
console.log(`HTTP status: ${startResult.status}`);
|
||||
console.log(`errors: ${JSON.stringify(startResult.json?.errors ?? startResult.json?.message ?? null)}`);
|
||||
reportCallAccounting();
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
@@ -55,82 +63,62 @@ async function main() {
|
||||
return;
|
||||
}
|
||||
|
||||
// ── Update 1 ───────────────────────────────────────────
|
||||
const prevQ1 = selectedQuestion;
|
||||
let update1Result = await postJson("/api/cases/update", {
|
||||
situationGraph,
|
||||
previousQuestion: prevQ1,
|
||||
answer: answers[0],
|
||||
});
|
||||
// ── Update loop (bounded, explicit, no retry) ──────────
|
||||
for (let i = 0; i < Math.min(config.maxUpdates, config.answers.length); i++) {
|
||||
const updateNum = i + 1;
|
||||
const prevQ = selectedQuestion;
|
||||
let updateResult = await postJson("/api/cases/update", {
|
||||
situationGraph,
|
||||
previousQuestion: prevQ,
|
||||
answer: config.answers[i],
|
||||
});
|
||||
calls.updateCalls++;
|
||||
|
||||
if (!update1Result.json?.success) {
|
||||
console.log("\n=== UPDATE 1 ===");
|
||||
console.log(`HTTP status: ${update1Result.status}`);
|
||||
console.log(`stage: ${update1Result.json?.stage ?? "unknown"}`);
|
||||
console.log(`selected question: null`);
|
||||
console.log(`node count: ${nodeCount(situationGraph)}`);
|
||||
console.log(`edge count: ${edgeCount(situationGraph)}`);
|
||||
console.log(`error/validation summary: ${JSON.stringify(update1Result.json?.errors ?? update1Result.json?.proposalErrors ?? update1Result.json?.graphValidationErrors ?? update1Result.json?.validationErrors ?? update1Result.json?.message ?? null)}`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
if (!updateResult.json?.success) {
|
||||
console.log(`\n=== UPDATE ${updateNum} ===`);
|
||||
console.log(`HTTP status: ${updateResult.status}`);
|
||||
console.log(`stage: ${updateResult.json?.stage ?? "unknown"}`);
|
||||
console.log(`selected question: null`);
|
||||
console.log(`node count: ${nodeCount(situationGraph)}`);
|
||||
console.log(`edge count: ${edgeCount(situationGraph)}`);
|
||||
console.log(`error/validation summary: ${JSON.stringify(updateResult.json?.errors ?? updateResult.json?.proposalErrors ?? updateResult.json?.graphValidationErrors ?? updateResult.json?.validationErrors ?? updateResult.json?.message ?? null)}`);
|
||||
|
||||
// Preserve rejectedProposalSnapshot diagnostics if present
|
||||
if (updateResult.json?.diagnostics?.rejectedProposalSnapshot) {
|
||||
console.log(`\ndiagnostics.rejectedProposalSnapshot: ${JSON.stringify(updateResult.json.diagnostics.rejectedProposalSnapshot, null, 2)}`);
|
||||
}
|
||||
|
||||
reportCallAccounting();
|
||||
console.log("\n*** UPDATE REJECTED — STOPPING (no retry). ***");
|
||||
process.exitCode = 1;
|
||||
return; // rejection stops the chain immediately
|
||||
}
|
||||
|
||||
const updatedGraph = updateResult.json.updatedSituationGraph;
|
||||
selectedQuestion = updateResult.json.selectedQuestion?.question ?? null;
|
||||
|
||||
console.log(`\n=== UPDATE ${updateNum} ===`);
|
||||
console.log(`HTTP status: ${updateResult.status}`);
|
||||
console.log(`stage: ${updateResult.json.stage ?? "unknown"}`);
|
||||
console.log(`proposal/apply success: ${updateResult.json.proposal?.success ?? updateResult.json.applySuccess ?? null}`);
|
||||
console.log(`selected question: ${JSON.stringify(selectedQuestion)}`);
|
||||
console.log(`node count: ${nodeCount(updatedGraph)}`);
|
||||
console.log(`edge count: ${edgeCount(updatedGraph)}`);
|
||||
|
||||
situationGraph = updatedGraph;
|
||||
}
|
||||
}
|
||||
|
||||
const updatedGraph1 = update1Result.json.updatedSituationGraph;
|
||||
selectedQuestion = update1Result.json.selectedQuestion?.question ?? null;
|
||||
|
||||
console.log("\n=== UPDATE 1 ===");
|
||||
console.log(`HTTP status: ${update1Result.status}`);
|
||||
console.log(`stage: ${update1Result.json.stage ?? "unknown"}`);
|
||||
console.log(`proposal/apply success: ${update1Result.json.proposal?.success ?? update1Result.json.applySuccess ?? null}`);
|
||||
console.log(`selected question: ${JSON.stringify(selectedQuestion)}`);
|
||||
console.log(`node count: ${nodeCount(updatedGraph1)}`);
|
||||
console.log(`edge count: ${edgeCount(updatedGraph1)}`);
|
||||
console.log(
|
||||
`error/validation summary: ${JSON.stringify(update1Result.json?.errors ?? update1Result.json?.proposalErrors ?? update1Result.json?.graphValidationErrors ?? update1Result.json?.validationErrors ?? null)}`
|
||||
);
|
||||
|
||||
if (!selectedQuestion) {
|
||||
console.log("ERROR: No selected question returned from update 1. Stopping.");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// ── Update 2 ───────────────────────────────────────────
|
||||
const prevQ2 = selectedQuestion;
|
||||
let update2Result = await postJson("/api/cases/update", {
|
||||
situationGraph: updatedGraph1,
|
||||
previousQuestion: prevQ2,
|
||||
answer: answers[1],
|
||||
});
|
||||
|
||||
if (!update2Result.json?.success) {
|
||||
console.log("\n=== UPDATE 2 ===");
|
||||
console.log(`HTTP status: ${update2Result.status}`);
|
||||
console.log(`stage: ${update2Result.json?.stage ?? "unknown"}`);
|
||||
console.log(`selected question: null`);
|
||||
console.log(`node count: ${nodeCount(updatedGraph1)}`);
|
||||
console.log(`edge count: ${edgeCount(updatedGraph1)}`);
|
||||
console.log(`error/validation summary: ${JSON.stringify(update2Result.json?.errors ?? update2Result.json?.proposalErrors ?? update2Result.json?.graphValidationErrors ?? update2Result.json?.validationErrors ?? update2Result.json?.message ?? null)}`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const updatedGraph2 = update2Result.json.updatedSituationGraph;
|
||||
selectedQuestion = update2Result.json.selectedQuestion?.question ?? null;
|
||||
|
||||
console.log("\n=== UPDATE 2 ===");
|
||||
console.log(`HTTP status: ${update2Result.status}`);
|
||||
console.log(`stage: ${update2Result.json.stage ?? "unknown"}`);
|
||||
console.log(`proposal/apply success: ${update2Result.json.proposal?.success ?? update2Result.json.applySuccess ?? null}`);
|
||||
console.log(`selected question: ${JSON.stringify(selectedQuestion)}`);
|
||||
console.log(`node count: ${nodeCount(updatedGraph2)}`);
|
||||
console.log(`edge count: ${edgeCount(updatedGraph2)}`);
|
||||
console.log(
|
||||
`error/validation summary: ${JSON.stringify(update2Result.json?.errors ?? update2Result.json?.proposalErrors ?? update2Result.json?.graphValidationErrors ?? update2Result.json?.validationErrors ?? null)}`
|
||||
);
|
||||
function reportCallAccounting() {
|
||||
const totalCalls = calls.startCalls + calls.updateCalls;
|
||||
console.log("\n=== CALL ACCOUNTING ===");
|
||||
console.log(`startCalls: ${calls.startCalls}`);
|
||||
console.log(`updateCalls: ${calls.updateCalls}`);
|
||||
console.log(`totalCalls: ${totalCalls}`);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
reportCallAccounting();
|
||||
process.exitCode = 1;
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user