tooling: add pre-anchored update-only mode to canonical harness
Add FIXTURE_MODE=updateOnly support that bypasses Start and sends the committed fixture (tests/fixtures/pre-anchored-update-savings-realism.json) directly as an Update request body through production HTTP route. scripts/reproduce-multi-turn-investigation.mjs: - Added ESM imports for deterministic fixture loading (fs, fileURLToPath, path) - Added FIXTURE_PATH constant pointing to committed fixture - Added fixtureMode env-var selector and runUpdateOnlyMode() function - Validates ANSWER_2 before any live call (zero calls if missing) - Verifies single savings-realism anchor invariant on load - Preserves all hardened capture fields in pre-anchored mode - Normal-mode Start→Update chain preserved under guard clause tests/reproduce-multi-turn-investigation.harness.test.js: - Added 7 new harness tests for pre-anchored scenarios (46 total, all pass) - Updated runPreAnchoredSimulation to persist rejectedProposalSnapshot on rejection - Added runPreAnchoredSimulationWithBlock() helper docs/: - New docs/experiment-57j78.md with full apparatus description - Updated docs/current-handoff.md with 57J.78 section
This commit is contained in:
@@ -1,3 +1,12 @@
|
||||
import fs from "fs";
|
||||
import { fileURLToPath } from "url";
|
||||
import path from "path";
|
||||
|
||||
const FIXTURE_PATH = path.resolve(
|
||||
path.dirname(fileURLToPath(import.meta.url)),
|
||||
"../tests/fixtures/pre-anchored-update-savings-realism.json",
|
||||
);
|
||||
|
||||
const BASE_URL =
|
||||
process.env.CONFIDENCE_ENGINE_BASE_URL || "http://127.0.0.1:3000";
|
||||
|
||||
@@ -12,6 +21,9 @@ const config = {
|
||||
],
|
||||
};
|
||||
|
||||
// ── Mode selector ────────────────────────────────────────
|
||||
const fixtureMode = process.env.FIXTURE_MODE;
|
||||
|
||||
// ── Call accounting (reflects actual API calls, not successes) ──
|
||||
const calls = { startCalls: 0, updateCalls: 0 };
|
||||
|
||||
@@ -34,7 +46,20 @@ function edgeCount(g) {
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// ── Start (exactly one call, no retry) ────────────────
|
||||
// ── Mode dispatch ──────────────────────────────────────
|
||||
if (fixtureMode === "updateOnly") {
|
||||
await runUpdateOnlyMode();
|
||||
reportCallAccounting();
|
||||
return;
|
||||
}
|
||||
|
||||
if (fixtureMode !== undefined) {
|
||||
console.error(`ERROR: Unsupported FIXTURE_MODE="${fixtureMode}". Use "updateOnly" or unset.`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// ── Normal mode: Start→Update chain (unchanged from original) ──
|
||||
const startResult = await postJson("/api/cases/start", { scenario: config.scenario });
|
||||
calls.startCalls++;
|
||||
|
||||
@@ -160,6 +185,156 @@ async function main() {
|
||||
}
|
||||
}
|
||||
|
||||
// ── Pre-anchored update-only mode (57J.78) ───────────────
|
||||
|
||||
/**
|
||||
* Loads the committed pre-anchored fixture, skips Start,
|
||||
* sends exactly one Update through the production HTTP route,
|
||||
* and preserves all hardened capture/no-retry behaviour.
|
||||
*/
|
||||
async function runUpdateOnlyMode() {
|
||||
const answer = process.env.ANSWER_2;
|
||||
|
||||
if (!answer || String(answer).trim() === "") {
|
||||
console.log("BLOCKED - missing ANSWER_2");
|
||||
return; // zero live calls made
|
||||
}
|
||||
|
||||
// Load committed fixture — single source of truth.
|
||||
let fixtureData;
|
||||
try {
|
||||
const raw = fs.readFileSync(FIXTURE_PATH, "utf-8");
|
||||
fixtureData = JSON.parse(raw);
|
||||
} catch (err) {
|
||||
console.error(`ERROR: Cannot load pre-anchored fixture from ${FIXTURE_PATH}`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const initialGraph = fixtureData.graph;
|
||||
|
||||
// Verify fixture integrity before proceeding.
|
||||
const nodes = initialGraph.nodes;
|
||||
const edges = initialGraph.edges;
|
||||
const savingsNodes = nodes.filter(
|
||||
(n) => n.kind === "unknown" && n.status === "unknown" && n.label.includes("savings"),
|
||||
);
|
||||
|
||||
if (savingsNodes.length !== 1) {
|
||||
console.log(`ERROR: pre-anchored fixture does not contain exactly one savings-realism anchor (found ${savingsNodes.length}).`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// Pre-call compact fixture snapshot.
|
||||
const savingsNode = savingsNodes[0];
|
||||
console.log(`\n=== UPDATE-ONLY MODE ===`);
|
||||
console.log(`fixture node count: ${nodes.length}`);
|
||||
console.log(`fixture edge count: ${edges.length}`);
|
||||
console.log(`savings-realism node:`);
|
||||
console.log(` id: ${savingsNode.id}`);
|
||||
console.log(` label: ${savingsNode.label}`);
|
||||
console.log(` kind: ${savingsNode.kind}`);
|
||||
console.log(` status: ${savingsNode.status}`);
|
||||
|
||||
// Deep-copy so mutation doesn't corrupt the original fixture.
|
||||
const situationGraph = JSON.parse(JSON.stringify(initialGraph));
|
||||
let selectedQuestion = null;
|
||||
|
||||
// ── Exactly one Update through production route ────────
|
||||
const updateNum = 1;
|
||||
let updateResult = await postJson("/api/cases/update", {
|
||||
situationGraph,
|
||||
previousQuestion: selectedQuestion,
|
||||
answer: String(answer),
|
||||
});
|
||||
calls.updateCalls++;
|
||||
|
||||
if (!updateResult.json?.success) {
|
||||
console.log(`\n=== UPDATE ${updateNum} ===`);
|
||||
console.log(`HTTP status: ${updateResult.status}`);
|
||||
console.log(`stage: ${updateResult.json?.stage ?? "unknown"}`);
|
||||
console.log(`selected question: null`);
|
||||
console.log(`node count: ${nodeCount(situationGraph)}`);
|
||||
console.log(`edge count: ${edgeCount(situationGraph)}`);
|
||||
console.log(
|
||||
`error/validation summary: ${JSON.stringify(updateResult.json?.errors ?? updateResult.json?.proposalErrors ?? updateResult.json?.graphValidationErrors ?? updateResult.json?.validationErrors ?? updateResult.json?.message ?? null)}`,
|
||||
);
|
||||
|
||||
// Preserve rejectedProposalSnapshot diagnostics if present.
|
||||
if (updateResult.json?.diagnostics?.rejectedProposalSnapshot) {
|
||||
console.log(
|
||||
`\ndiagnostics.rejectedProposalSnapshot: ${JSON.stringify(updateResult.json.diagnostics.rejectedProposalSnapshot, null, 2)}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Capture structuralActionRequired from rejected snapshot.
|
||||
const rejectedSnapshot = updateResult.json?.diagnostics?.rejectedProposalSnapshot ?? null;
|
||||
if (rejectedSnapshot && "structuralActionRequired" in rejectedSnapshot) {
|
||||
console.log(
|
||||
`structuralActionRequired (from rejected proposal snapshot): ${JSON.stringify(rejectedSnapshot.structuralActionRequired)}`,
|
||||
);
|
||||
} else {
|
||||
console.log(`structuralActionRequired: UNAVAILABLE`);
|
||||
}
|
||||
|
||||
console.log("\n*** UPDATE REJECTED — STOPPING (no retry). ***");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const updatedGraph = updateResult.json.updatedSituationGraph;
|
||||
selectedQuestion = updateResult.json.selectedQuestion?.question ?? null;
|
||||
|
||||
// ── Capture accepted answer meaning ─────────────────────
|
||||
const am = updateResult.json.answerMeaning ?? null;
|
||||
if (am) {
|
||||
console.log(`answerMeaning.userSupportedMeaning: ${JSON.stringify(am.userSupportedMeaning ?? null)}`);
|
||||
console.log(`answerMeaning.possibleInference: ${JSON.stringify(am.possibleInference ?? null)}`);
|
||||
console.log(`answerMeaning.supportCategory: ${JSON.stringify(am.supportCategory ?? null)}`);
|
||||
console.log(`answerMeaning.resolutionGuidance: ${JSON.stringify(am.resolutionGuidance ?? null)}`);
|
||||
}
|
||||
|
||||
// ── Capture accepted structural mutation fields ────────
|
||||
const proposal = updateResult.json.updatedProposal ?? updateResult.json.proposal ?? null;
|
||||
if (proposal) {
|
||||
console.log(`updatedNodes: ${JSON.stringify(proposal.updatedNodes ?? [])}`);
|
||||
console.log(`resolvedUnknownNodeIds: ${JSON.stringify(proposal.resolvedUnknownNodeIds ?? [])}`);
|
||||
console.log(`addedNodes: ${JSON.stringify(proposal.addedNodes ?? [])}`);
|
||||
console.log(`addedEdges: ${JSON.stringify(proposal.addedEdges ?? [])}`);
|
||||
}
|
||||
|
||||
// ── Capture structuralActionRequired from accepted update ─
|
||||
const sar = updateResult.json.structuralActionRequired;
|
||||
if (sar === undefined || sar === null) {
|
||||
console.log(`structuralActionRequired: null`);
|
||||
} else {
|
||||
console.log(`structuralActionRequired: ${JSON.stringify(sar)}`);
|
||||
}
|
||||
|
||||
// ── Capture selectedQuestion node reference ────────────
|
||||
const sq = updateResult.json.selectedQuestion ?? null;
|
||||
if (sq && typeof sq === "object") {
|
||||
console.log(`selectedQuestion: ${JSON.stringify(sq.question ?? null)}`);
|
||||
if (sq.nodeId) {
|
||||
console.log(`selectedQuestion.nodeId: ${JSON.stringify(sq.nodeId)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Capture persistent graph after update ───────────────
|
||||
const pNodes = updatedGraph?.nodes ?? [];
|
||||
const pEdges = updatedGraph?.edges ?? [];
|
||||
console.log(`\nresulting persistent graph (${pNodes.length} nodes, ${pEdges.length} edges):`);
|
||||
for (const n of pNodes) {
|
||||
console.log(` node: id=${n.id ?? n.nodeId}, kind=${n.kind}, label=${n.label ?? n.description ?? ""}, status=${n.status}`);
|
||||
}
|
||||
for (const e of pEdges) {
|
||||
console.log(` edge: from=${e.fromNodeId ?? e.from}, to=${e.toNodeId ?? e.to}, relationship=${e.relationship}`);
|
||||
}
|
||||
|
||||
// ── UPDATE-ONLY EXIT — no retry, no additional calls ───
|
||||
}
|
||||
|
||||
function reportCallAccounting() {
|
||||
const totalCalls = calls.startCalls + calls.updateCalls;
|
||||
console.log("\n=== CALL ACCOUNTING ===");
|
||||
|
||||
Reference in New Issue
Block a user