From d77a1ff04d38296440ee78f62ff65333045cb796 Mon Sep 17 00:00:00 2001 From: robbond Date: Wed, 12 Aug 2026 09:11:20 +0100 Subject: [PATCH] tooling: add pre-anchored update fixture --- .../pre-anchored-update-savings-realism.json | 60 +++ ...e-multi-turn-investigation.harness.test.js | 425 ++++++++++++++++++ 2 files changed, 485 insertions(+) create mode 100644 tests/fixtures/pre-anchored-update-savings-realism.json diff --git a/tests/fixtures/pre-anchored-update-savings-realism.json b/tests/fixtures/pre-anchored-update-savings-realism.json new file mode 100644 index 0000000..c39ed41 --- /dev/null +++ b/tests/fixtures/pre-anchored-update-savings-realism.json @@ -0,0 +1,60 @@ +{ + "description": "Deterministic pre-existing graph fixture for false/no-op update testing.", + "scenario": "We are considering relocating the engineering team to reduce operating costs.", + "unresolved_question": "Are the projected office savings from relocation realistic?", + "graph": { + "centralStatement": "We are considering relocating the engineering team to reduce operating costs.", + "nodes": [ + { + "id": "n_relocation_state", + "label": "Engineering team relocation consideration", + "description": "Current state: organisation is considering relocating its engineering team from London to Manchester to reduce operating costs.", + "kind": "state", + "status": "provisional", + "confidence": "high", + "value": null, + "unit": null, + "evidenceIds": [], + "dependsOn": [], + "affects": [], + "parentId": null, + "childIds": [] + }, + { + "id": "n_savings_realism", + "label": "Are the projected office savings from relocation realistic?", + "description": "Uncertainty: whether the projected office savings from the relocation are realistic.", + "kind": "unknown", + "status": "unknown", + "confidence": "low", + "value": null, + "unit": null, + "evidenceIds": [], + "dependsOn": ["n_relocation_state"], + "affects": [], + "parentId": null, + "childIds": [] + } + ], + "edges": [ + { + "id": "e-sr-to-state", + "fromNodeId": "n_savings_realism", + "toNodeId": "n_relocation_state", + "relationship": "depends_on", + "confidence": "medium", + "description": "savings-realism uncertainty depends on the relocation consideration state" + } + ], + "activeUnknownNodeId": "n_savings_realism", + "resolvedNodeIds": [], + "currentSummary": "Engineering team is being considered for relocation; savings realism uncertain.", + "reasoningState": { + "comparabilityStatus": null, + "relationshipStatus": null, + "relationshipAssessed": false, + "contradictionReasoningAllowed": true, + "reasoningStages": [] + } + } +} diff --git a/tests/reproduce-multi-turn-investigation.harness.test.js b/tests/reproduce-multi-turn-investigation.harness.test.js index 3e0a841..ef027d7 100644 --- a/tests/reproduce-multi-turn-investigation.harness.test.js +++ b/tests/reproduce-multi-turn-investigation.harness.test.js @@ -569,8 +569,433 @@ describe("reproduce-multi-turn-investigation harness: one-shot semantics", () => expect(rSuccess.startCalls).toBe(1); expect(rSuccess.apiLog.length).toBe(3); // 1 start + 2 updates, no extras }); + + // ── 57J.74: pre-anchored update-only fixture tests ─────────────────────── + + it("pre-anchored fixture contains exactly one unresolved savings-realism anchor", () => { + const graph = PRE_ANCHORED_FIXTURE.graph; + const savingsNodes = graph.nodes.filter((n) => n.kind === "unknown" && n.status === "unknown" && n.label.includes("savings")); + expect(savingsNodes.length).toBe(1); + expect(savingsNodes[0].id).toBe("n_savings_realism"); + }); + + it("pre-anchored fixture uses valid existing graph schema", () => { + const g = PRE_ANCHORED_FIXTURE.graph; + + // Graph-level fields + expect(typeof g.centralStatement).toBe("string"); + expect(g.centralStatement.length).toBeGreaterThan(0); + expect(Array.isArray(g.nodes)).toBe(true); + expect(g.nodes.length).toBeGreaterThanOrEqual(1); + expect(Array.isArray(g.edges)).toBe(true); + expect(typeof g.currentSummary).toBe("string"); + expect(g.currentSummary.length).toBeGreaterThan(0); + expect(g.activeUnknownNodeId).not.toBeNull(); + + // Node fields — each node must have valid kind/status/confidence + const validKinds = ["observation", "reported_claim", "metric", "state", "transition", "relationship", "assumption", "unknown", "conclusion"]; + const validStatuses = ["known", "unknown", "provisional", "supported", "weakened", "contradicted", "resolved"]; + const validConfidences = ["low", "medium", "high"]; + + for (const node of g.nodes) { + expect(validKinds).toContain(node.kind); + expect(validStatuses).toContain(node.status); + expect(validConfidences).toContain(node.confidence); + expect(typeof node.id).toBe("string"); + expect(node.id.length).toBeGreaterThan(0); + expect(typeof node.label).toBe("string"); + expect(node.label.length).toBeGreaterThan(0); + expect(typeof node.description).toBe("string"); + expect(node.description.length).toBeGreaterThan(0); + } + + // Edge fields + for (const edge of g.edges) { + const validRels = ["supports", "weakens", "contradicts", "depends_on", "causes", "may_cause", "measures", "compares_with", "updates", "other"]; + expect(validRels).toContain(edge.relationship); + expect(typeof edge.id).toBe("string"); + expect(edge.id.length).toBeGreaterThan(0); + expect(typeof edge.fromNodeId).toBe("string"); + expect(edge.fromNodeId.length).toBeGreaterThan(0); + expect(typeof edge.toNodeId).toBe("string"); + expect(edge.toNodeId.length).toBeGreaterThan(0); + expect(typeof edge.description).toBe("string"); + expect(edge.description.length).toBeGreaterThan(0); + } + }); + + it("pre-anchored fixture contains a valid relationship into the graph", () => { + const g = PRE_ANCHORED_FIXTURE.graph; + const srNode = g.nodes.find((n) => n.id === "n_savings_realism"); + expect(srNode).toBeDefined(); + + const edgesFromSr = g.edges.filter((e) => e.fromNodeId === srNode.id); + expect(edgesFromSr.length).toBeGreaterThan(0); + + const edge = edgesFromSr[0]; + // The edge connects the unknown into the state node + expect(edge.relationship).toBe("depends_on"); + + // Both nodes referenced by the edge must exist in the graph + const fromNode = g.nodes.find((n) => n.id === edge.fromNodeId); + const toNode = g.nodes.find((n) => n.id === edge.toNodeId); + expect(fromNode).toBeDefined(); + expect(toNode).toBeDefined(); + + // The edge must reference the savings-realism node as from + expect(fromNode.id).toBe("n_savings_realism"); + }); + + it("pre-anchored update-only mode sends the exact fixture graph into the real update request shape", () => { + const r = runPreAnchoredSimulation({ answer: "I am unsure whether the projected office savings from the relocation are realistic." }); + expect(r.startCalls).toBe(0); + expect(r.updateCalls).toBe(1); + + // Verify the captured graph matches what was sent + const { nodes, edges } = r; + const fixtureGraph = PRE_ANCHORED_FIXTURE.graph; + expect(nodes.length).toBe(fixtureGraph.nodes.length); + expect(edges.length).toBe(fixtureGraph.edges.length); + expect(nodes[0].id).toBe(fixtureGraph.nodes[0].id); + expect(nodes[1].id).toBe(fixtureGraph.nodes[1].id); + }); + + it("pre-anchored update-only mode does not call Start", () => { + const r = runPreAnchoredSimulation(); + expect(r.startCalls).toBe(0); + expect(r.updateCalls).toBe(1); + + const startEntries = r.apiLog.filter((e) => e.step === "start"); + const updateEntries = r.apiLog.filter((e) => e.step === "update"); + expect(startEntries.length).toBe(0); + expect(updateEntries.length).toBe(1); + }); + + it("pre-anchored update-only mode makes exactly one Update call", () => { + const r = runPreAnchoredSimulation(); + expect(r.updateCalls).toBe(1); + expect(r.type).toBe("all_success"); + + const updateEntries = r.apiLog.filter((e) => e.step === "update"); + expect(updateEntries.length).toBe(1); + }); + + it("normal Start→Update harness mode remains unchanged", () => { + // Verify existing normal mode still works with exactly 1 start call + const rNormal = runSimulation({ scenario: "test_ok", maxUpdates: 1, answers: ["good answer"] }); + expect(rNormal.startCalls).toBe(1); + expect(rNormal.updateCalls).toBeGreaterThanOrEqual(0); + + // Normal mode should not be affected by the new pre-anchored code paths + const rWithShape = runSimulationWithResponseShape({ + scenario: "test_ok", + maxUpdates: 1, + answers: ["good answer"], + onResponseUpdate: () => ({ + success: true, + stage: "update_applied", + updatedSituationGraph: { nodes: [], edges: [] }, + selectedQuestion: { question: "q2" }, + updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] }, + }), + }); + expect(rWithShape.startCalls).toBe(1); + expect(rWithShape.updateCalls).toBe(1); + }); + + it("57J.62 accepted/rejected capture hardening remains unchanged", () => { + // Capture addedNodes via accepted response shape (existing test pattern) + const rAccepted = runSimulationWithResponseShape({ + scenario: "test_ok", + maxUpdates: 1, + answers: ["good answer"], + onResponseUpdate: () => ({ + success: true, + stage: "update_applied", + updatedSituationGraph: { nodes: [], edges: [] }, + selectedQuestion: { question: "q2" }, + updatedProposal: { + addedNodes: [{ id: "n_test", kind: "unknown", label: "test", description: "test", confidence: "low", value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], parentId: null, childIds: [] }], + addedEdges: [], + updatedNodes: [{ nodeId: "n_existing", newValue: "confirmed" }], + resolvedUnknownNodeIds: ["n_resolved"], + }, + }), + }); + expect(rAccepted.captured.proposal.addedNodes.length).toBe(1); + expect(rAccepted.captured.proposal.updatedNodes.length).toBe(1); + expect(rAccepted.captured.proposal.resolvedUnknownNodeIds.length).toBe(1); + + // Capture rejected snapshot via rejection (existing test pattern) + const rRejected = runSimulation({ scenario: "test_ok", maxUpdates: 1, answers: ["answer_reject"] }); + expect(rRejected.type).toBe("update_rejection"); + expect(rRejected.startCalls).toBe(1); + expect(rRejected.updateCalls).toBe(1); + }); + + it("57J.72 direct structuralActionRequired capture remains unchanged", () => { + // Accepted update with true (existing test pattern) + const rTrue = runSimulationWithResponseShape({ + scenario: "test_ok", + maxUpdates: 1, + answers: ["good answer"], + onResponseUpdate: () => ({ + success: true, + stage: "update_applied", + updatedSituationGraph: { nodes: [], edges: [] }, + selectedQuestion: { question: "q2" }, + structuralActionRequired: true, + updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] }, + }), + }); + expect(rTrue.captured.structuralActionRequired).toBe(true); + + // Accepted update with false (existing test pattern) + const rFalse = runSimulationWithResponseShape({ + scenario: "test_ok", + maxUpdates: 1, + answers: ["good answer"], + onResponseUpdate: () => ({ + success: true, + stage: "update_applied", + updatedSituationGraph: { nodes: [], edges: [] }, + selectedQuestion: { question: "q2" }, + structuralActionRequired: false, + updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] }, + }), + }); + expect(rFalse.captured.structuralActionRequired).toBe(false); + + // Absent field reports null (existing test pattern) + const rAbsent = runSimulationWithResponseShape({ + scenario: "test_ok", + maxUpdates: 1, + answers: ["good answer"], + onResponseUpdate: () => ({ + success: true, + stage: "update_applied", + updatedSituationGraph: { nodes: [], edges: [] }, + selectedQuestion: { question: "q2" }, + updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] }, + }), + }); + expect(rAbsent.captured.structuralActionRequired).toBeNull(); + + // Rejected snapshot (existing test pattern) + const rSnapshot = runSimulationWithRejectionSnapshot({ + scenario: "test_ok", + maxUpdates: 1, + answers: ["answer_reject"], + rejectedSnapshot: { structuralActionRequired: true }, + }); + expect(rSnapshot.rejectedSnapshot.structuralActionRequired).toBe(true); + }); + + it("no retries or supplementary calls are introduced in pre-anchored mode", () => { + const r = runPreAnchoredSimulation(); + const totalCalls = r.startCalls + r.updateCalls; + expect(totalCalls).toBe(1); // 0 start + 1 update only + expect(r.apiLog.length).toBe(1); + expect(r.type).toBe("all_success"); + expect(r.exitCode).toBe(0); + + const retryEntries = r.apiLog.filter((e) => e.step === "update" && e.answer?.includes("_retry")); + expect(retryEntries.length).toBe(0); + }); }); +// ── Pre-anchored update-only mode (57J.74) ───────────── + +/** + * Deterministic fixture used by pre-anchored tests. + * Scenario: considering relocating engineering team; savings-realism unknown already present. + */ +const PRE_ANCHORED_FIXTURE = { + description: "Deterministic pre-existing graph fixture for false/no-op update testing.", + scenario: "We are considering relocating the engineering team to reduce operating costs.", + unresolvedQuestion: "Are the projected office savings from relocation realistic?", + graph: { + centralStatement: + "We are considering relocating the engineering team to reduce operating costs.", + nodes: [ + { + id: "n_relocation_state", + label: "Engineering team relocation consideration", + description: + "Current state: organisation is considering relocating its engineering team from London to Manchester to reduce operating costs.", + kind: "state", + status: "provisional", + confidence: "high", + value: null, + unit: null, + evidenceIds: [], + dependsOn: [], + affects: [], + parentId: null, + childIds: [], + }, + { + id: "n_savings_realism", + label: "Are the projected office savings from relocation realistic?", + description: + "Uncertainty: whether the projected office savings from the relocation are realistic.", + kind: "unknown", + status: "unknown", + confidence: "low", + value: null, + unit: null, + evidenceIds: [], + dependsOn: ["n_relocation_state"], + affects: [], + parentId: null, + childIds: [], + }, + ], + edges: [ + { + id: "e-sr-to-state", + fromNodeId: "n_savings_realism", + toNodeId: "n_relocation_state", + relationship: "depends_on", + confidence: "medium", + description: + "savings-realism uncertainty depends on the relocation consideration state", + }, + ], + activeUnknownNodeId: "n_savings_realism", + resolvedNodeIds: [], + currentSummary: + "Engineering team is being considered for relocation; savings realism uncertain.", + reasoningState: { + comparabilityStatus: null, + relationshipStatus: null, + relationshipAssessed: false, + contradictionReasoningAllowed: true, + reasoningStages: [], + }, + }, +}; + +/** + * Synchronous simulator of the pre-anchored update-only harness mode. + * Mirrors what reproduce-multi-turn-investigation.mjs does when fixtureMode is "updateOnly". + */ +function runPreAnchoredSimulation(cfg) { + let startCalls = 0; + let updateCalls = 0; + let apiLog = []; + + const initialGraph = cfg?.initialGraph || PRE_ANCHORED_FIXTURE.graph; + const answer = cfg?.answer || "I am unsure whether the projected office savings from the relocation are realistic."; + + // Normalise cfg to safe defaults so the closure always reads non-null fields. + const C = { ...cfg, initialGraph: undefined, answer: undefined }; + + // Pre-anchored: no Start call — graph is supplied directly. + let graph = JSON.parse(JSON.stringify(initialGraph)); + let question = null; // will be set by the update response (or mock) + + const api = { + post(path, body) { + if (path === "/api/cases/start") { + apiLog.push({ step: "start" }); + startCalls++; + return { status: 200, json: () => ({ success: true, situationGraph: { nodes: [], edges: [] }, selectedQuestion: { question: "q" } }) }; + } + + if (path === "/api/cases/update") { + apiLog.push({ step: "update", answer: body.answer }); + updateCalls++; + + const resp = typeof C.onResponseUpdate === "function" + ? C.onResponseUpdate(updateCalls - 1) + : null; + + if (resp) { + return { status: resp.success ? 200 : 422, json: () => resp }; + } + + // Default pre-anchored success response + const updatedNodes = C.updateBehavior === "no-op" + ? [{ nodeId: graph.nodes[1].id, newValue: null }] + : []; + + return { + status: 200, + json: () => ({ + success: true, + stage: "update_applied", + updatedSituationGraph: graph, + selectedQuestion: { question: "What evidence would clarify projected savings realism?", nodeId: "n_savings_realism" }, + structuralActionRequired: C.updateBehavior === "no-op" ? false : true, + answerMeaning: { + userSupportedMeaning: "User is unsure whether the projected office savings from the relocation are realistic.", + possibleInference: null, + supportCategory: "uncertain", + resolutionGuidance: null, + }, + updatedProposal: { + addedNodes: C.updateBehavior === "no-op" ? [] : [{ id: "n_new_unknown", kind: "unknown", label: "test node", description: "test", confidence: "low", value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], parentId: null, childIds: [] }], + addedEdges: C.updateBehavior === "no-op" ? [] : [{ id: "e-new", fromNodeId: "n_new_unknown", toNodeId: "n_relocation_state", relationship: "depends_on", confidence: "medium", description: "test edge" }], + updatedNodes: updatedNodes, + resolvedUnknownNodeIds: [], + }, + }), + }; + } + + apiLog.push({ step: "unknown", path }); + return { status: 404, json: () => ({ error: "not found" }) }; + }, + }; + + // ── Pre-anchored: no Start call. Use supplied graph directly. ── + + // Verify fixture integrity before proceeding (what harness would report) + const nodes = initialGraph.nodes; + const edges = initialGraph.edges; + const savingsNodes = nodes.filter((n) => n.kind === "unknown" && n.status === "unknown" && n.label.includes("savings")); + + if (cfg?.requireAnchor !== false && savingsNodes.length !== 1) { + return { + startCalls, updateCalls, type: "anchor_validation_failed", exitCode: 1, + anchorCount: savingsNodes.length, apiLog, nodes, edges, + }; + } + + // Pre-anchored mode sends the exact fixture graph into the update request. + const upResp = api.post("/api/cases/update", { + situationGraph: graph, + previousQuestion: question, + answer, + }); + const uj = upResp.json(); + + if (!uj.success) { + return { + startCalls, updateCalls, type: "update_rejection", exitCode: 1, apiLog, nodes, edges, + }; + } + + // Capture fields (mirrors harness print logic) + const am = uj.answerMeaning ?? null; + let proposal = uj.updatedProposal ?? uj.proposal ?? null; + let sar = uj.structuralActionRequired; + if (sar === undefined || sar === null) sar = null; + const sq = uj.selectedQuestion ?? null; + + return { + startCalls, updateCalls, type: "all_success", exitCode: 0, apiLog, nodes, edges, + captured: { + answerMeaning: am ? { userSupportedMeaning: am.userSupportedMeaning, possibleInference: am.possibleInference, supportCategory: am.supportCategory, resolutionGuidance: am.resolutionGuidance } : null, + proposal: proposal ? { addedNodes: proposal.addedNodes ?? [], addedEdges: proposal.addedEdges ?? [], updatedNodes: proposal.updatedNodes ?? [], resolvedUnknownNodeIds: proposal.resolvedUnknownNodeIds ?? [] } : null, + structuralActionRequired: sar, + selectedQuestion: sq && typeof sq === "object" ? { question: sq.question, nodeId: sq.nodeId } : null, + graph: { nodes: uj.updatedSituationGraph?.nodes ?? [], edges: uj.updatedSituationGraph?.edges ?? [] }, + }, + }; +} + /** * Extended simulation with capture recording — mirrors what the updated harness prints. */