From c4431997b1c00a5d5b743d07f2b4d5f0c3101955 Mon Sep 17 00:00:00 2001 From: robbond Date: Wed, 12 Aug 2026 12:00:18 +0100 Subject: [PATCH] tooling: capture accepted answer meaning directly --- .../reproduce-multi-turn-investigation.mjs | 11 +- ...e-multi-turn-investigation.harness.test.js | 100 ++++++++++-------- 2 files changed, 60 insertions(+), 51 deletions(-) diff --git a/scripts/reproduce-multi-turn-investigation.mjs b/scripts/reproduce-multi-turn-investigation.mjs index eb4f1b5..53ebbb4 100644 --- a/scripts/reproduce-multi-turn-investigation.mjs +++ b/scripts/reproduce-multi-turn-investigation.mjs @@ -135,8 +135,11 @@ async function main() { console.log(`stage: ${updateResult.json.stage ?? "unknown"}`); console.log(`proposal/apply success: ${updateResult.json.proposal?.success ?? updateResult.json.applySuccess ?? null}`); - // ── Capture accepted answer meaning ───────────────────── - const am = updateResult.json.answerMeaning ?? null; + // ── Capture accepted structural mutation fields ──────── + const proposal = updateResult.json.updatedProposal ?? updateResult.json.proposal ?? null; + + // ── Capture answerMeaning and structuralActionRequired from proposal (production path) ─ + const am = proposal?.answerMeaning ?? null; if (am) { console.log(`answerMeaning.userSupportedMeaning: ${JSON.stringify(am.userSupportedMeaning ?? null)}`); console.log(`answerMeaning.possibleInference: ${JSON.stringify(am.possibleInference ?? null)}`); @@ -144,8 +147,6 @@ async function main() { console.log(`answerMeaning.resolutionGuidance: ${JSON.stringify(am.resolutionGuidance ?? null)}`); } - // ── Capture accepted structural mutation fields ──────── - const proposal = updateResult.json.updatedProposal ?? updateResult.json.proposal ?? null; if (proposal) { console.log(`updatedNodes: ${JSON.stringify(proposal.updatedNodes ?? [])}`); console.log(`resolvedUnknownNodeIds: ${JSON.stringify(proposal.resolvedUnknownNodeIds ?? [])}`); @@ -154,7 +155,7 @@ async function main() { } // ── Capture structuralActionRequired from accepted update ─ - const sar = updateResult.json.structuralActionRequired; + const sar = proposal?.structuralActionRequired; if (sar === undefined || sar === null) { console.log(`structuralActionRequired: null`); } else { diff --git a/tests/reproduce-multi-turn-investigation.harness.test.js b/tests/reproduce-multi-turn-investigation.harness.test.js index f872785..fc00adf 100644 --- a/tests/reproduce-multi-turn-investigation.harness.test.js +++ b/tests/reproduce-multi-turn-investigation.harness.test.js @@ -272,13 +272,7 @@ describe("reproduce-multi-turn-investigation harness: one-shot semantics", () => stage: "update_applied", updatedSituationGraph: { nodes: [], edges: [] }, selectedQuestion: { question: "q2" }, - answerMeaning: { - userSupportedMeaning: "cost reduction is a primary driver", - possibleInference: "if cost savings not realized, relocation weakens", - supportCategory: "other", - resolutionGuidance: "determine actual projected figures", - }, - updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] }, + updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [], answerMeaning: { userSupportedMeaning: "cost reduction is a primary driver", possibleInference: "if cost savings not realized, relocation weakens", supportCategory: "other", resolutionGuidance: "determine actual projected figures" } }, }), }); expect(r.updateCalls).toBe(1); @@ -389,8 +383,7 @@ describe("reproduce-multi-turn-investigation harness: one-shot semantics", () => stage: "update_applied", updatedSituationGraph: { nodes: [], edges: [] }, selectedQuestion: { question: "q2" }, - structuralActionRequired: true, - updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] }, + updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [], structuralActionRequired: true }, }), }); expect(r.startCalls).toBe(1); @@ -408,8 +401,7 @@ describe("reproduce-multi-turn-investigation harness: one-shot semantics", () => stage: "update_applied", updatedSituationGraph: { nodes: [], edges: [] }, selectedQuestion: { question: "q2" }, - structuralActionRequired: false, - updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] }, + updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [], structuralActionRequired: false }, }), }); expect(r.updateCalls).toBe(1); @@ -444,8 +436,7 @@ describe("reproduce-multi-turn-investigation harness: one-shot semantics", () => stage: "update_applied", updatedSituationGraph: { nodes: [], edges: [] }, selectedQuestion: { question: "q2" }, - structuralActionRequired: null, - updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] }, + updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [], structuralActionRequired: null }, }), }); expect(r.updateCalls).toBe(1); @@ -561,8 +552,7 @@ describe("reproduce-multi-turn-investigation harness: one-shot semantics", () => stage: "update_applied", updatedSituationGraph: { nodes: [], edges: [] }, selectedQuestion: { question: "q" }, - structuralActionRequired: true, - updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] }, + updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [], structuralActionRequired: true }, }), }); expect(rSuccess.updateCalls).toBe(2); @@ -863,18 +853,18 @@ describe("reproduce-multi-turn-investigation harness: one-shot semantics", () => edges: [{ fromNodeId: "n_proj_validation", toNodeId: "n_relocation_state", relationship: "depends_on" }], }, selectedQuestion: { question: "What evidence would clarify validation?", nodeId: "n_proj_validation" }, - structuralActionRequired: true, - answerMeaning: { - userSupportedMeaning: "User is unsure whether projected savings are realistic.", - possibleInference: null, - supportCategory: "uncertain", - resolutionGuidance: null, - }, updatedProposal: { addedNodes: [{ id: "n_proj_validation" }], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [], + structuralActionRequired: true, + answerMeaning: { + userSupportedMeaning: "User is unsure whether projected savings are realistic.", + possibleInference: null, + supportCategory: "uncertain", + resolutionGuidance: null, + }, }, }), }); @@ -1141,18 +1131,19 @@ function runPreAnchoredSimulation(cfg) { stage: "update_applied", updatedSituationGraph: graph, selectedQuestion: { question: "What evidence would clarify projected savings realism?", nodeId: "n_savings_realism" }, - structuralActionRequired: C.updateBehavior === "no-op" ? false : true, - answerMeaning: { - userSupportedMeaning: "User is unsure whether the projected office savings from the relocation are realistic.", - possibleInference: null, - supportCategory: "uncertain", - resolutionGuidance: null, - }, + // structuralActionRequired and answerMeaning only inside updatedProposal (production path) updatedProposal: { addedNodes: C.updateBehavior === "no-op" ? [] : [{ id: "n_new_unknown", kind: "unknown", label: "test node", description: "test", confidence: "low", value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], parentId: null, childIds: [] }], addedEdges: C.updateBehavior === "no-op" ? [] : [{ id: "e-new", fromNodeId: "n_new_unknown", toNodeId: "n_relocation_state", relationship: "depends_on", confidence: "medium", description: "test edge" }], updatedNodes: updatedNodes, resolvedUnknownNodeIds: [], + structuralActionRequired: C.updateBehavior === "no-op" ? false : true, + answerMeaning: { + userSupportedMeaning: "User is unsure whether the projected office savings from the relocation are realistic.", + possibleInference: null, + supportCategory: "uncertain", + resolutionGuidance: null, + }, }, }), }; @@ -1191,10 +1182,10 @@ function runPreAnchoredSimulation(cfg) { }; } - // Capture fields (mirrors harness print logic) - const am = uj.answerMeaning ?? null; + // Capture fields — mirrors harness production path (from proposal, not root) let proposal = uj.updatedProposal ?? uj.proposal ?? null; - let sar = uj.structuralActionRequired; + const am = proposal?.answerMeaning ?? null; + let sar = proposal?.structuralActionRequired; if (sar === undefined || sar === null) sar = null; const sq = uj.selectedQuestion ?? null; @@ -1238,10 +1229,26 @@ function runSimulationWithResponseShape(cfg) { : null; if (resp) { - return { status: resp.success ? 200 : 422, json: () => resp }; + // Production exposes these inside updatedProposal; read from there. + const proposalFromResp = resp.updatedProposal ?? {}; + const sar = proposalFromResp.structuralActionRequired !== undefined + ? proposalFromResp.structuralActionRequired + : (resp.structuralActionRequired !== undefined ? resp.structuralActionRequired : undefined); + const am = proposalFromResp.answerMeaning ?? resp.answerMeaning ?? null; + return { status: resp.success ? 200 : 422, json: () => ({ + ...resp, + updatedProposal: { + addedNodes: proposalFromResp.addedNodes ?? [], + addedEdges: proposalFromResp.addedEdges ?? [], + updatedNodes: proposalFromResp.updatedNodes ?? [], + resolvedUnknownNodeIds: proposalFromResp.resolvedUnknownNodeIds ?? [], + structuralActionRequired: sar, + answerMeaning: am, + }, + }) }; } - // Fallback to default mock + // Fallback to default mock (no structuralActionRequired or answerMeaning at any level) const shouldReject = typeof body.answer === "string" && body.answer.includes("_reject"); return { status: shouldReject ? 422 : 200, @@ -1281,21 +1288,22 @@ function runSimulationWithResponseShape(cfg) { graph = uj.updatedSituationGraph; question = uj.selectedQuestion ? uj.selectedQuestion.question : null; - // Mirror the harness capture (what would be printed) - const am = uj.answerMeaning ?? null; - if (am) captured.answerMeaning = { userSupportedMeaning: am.userSupportedMeaning, possibleInference: am.possibleInference, supportCategory: am.supportCategory, resolutionGuidance: am.resolutionGuidance }; - + // Mirror the harness capture — reads from proposal (production path), NOT root. const proposal = uj.updatedProposal ?? uj.proposal ?? null; - if (proposal) captured.proposal = { addedNodes: proposal.addedNodes ?? [], addedEdges: proposal.addedEdges ?? [], updatedNodes: proposal.updatedNodes ?? [], resolvedUnknownNodeIds: proposal.resolvedUnknownNodeIds ?? [] }; - - // Mirror the harness structuralActionRequired capture for accepted updates - const sar = uj.structuralActionRequired; - if (sar === undefined || sar === null) { - captured.structuralActionRequired = null; - } else { - captured.structuralActionRequired = sar; + if (proposal) { + captured.proposal = { addedNodes: proposal.addedNodes ?? [], addedEdges: proposal.addedEdges ?? [], updatedNodes: proposal.updatedNodes ?? [], resolvedUnknownNodeIds: proposal.resolvedUnknownNodeIds ?? [] }; + // Structural fields live inside graphUpdate schema, not at root level. + const sar = proposal.structuralActionRequired; + if (sar === undefined || sar === null) { + captured.structuralActionRequired = null; + } else { + captured.structuralActionRequired = sar; + } } + const am = proposal?.answerMeaning ?? null; + if (am) captured.answerMeaning = { userSupportedMeaning: am.userSupportedMeaning, possibleInference: am.possibleInference, supportCategory: am.supportCategory, resolutionGuidance: am.resolutionGuidance }; + const sq = uj.selectedQuestion ?? null; if (sq && typeof sq === "object") captured.selectedQuestion = { question: sq.question, nodeId: sq.nodeId };