robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tooling: capture accepted answer meaning directly

Browse Source
This commit is contained in:
robbond
2026-08-12 12:00:18 +01:00
parent d2891730af
commit c4431997b1
2 changed files with 60 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/reproduce-multi-turn-investigation.mjs
+6 -5
View File
@@ -135,8 +135,11 @@ async function main() {
console.log(`stage: ${updateResult.json.stage ?? "unknown"}`);
console.log(`proposal/apply success: ${updateResult.json.proposal?.success ?? updateResult.json.applySuccess ?? null}`);
// ── Capture accepted answer meaning ─────────────────────
const am = updateResult.json.answerMeaning ?? null;
// ── Capture accepted structural mutation fields ────────
const proposal = updateResult.json.updatedProposal ?? updateResult.json.proposal ?? null;
// ── Capture answerMeaning and structuralActionRequired from proposal (production path) ─
const am = proposal?.answerMeaning ?? null;
if (am) {
console.log(`answerMeaning.userSupportedMeaning: ${JSON.stringify(am.userSupportedMeaning ?? null)}`);
console.log(`answerMeaning.possibleInference: ${JSON.stringify(am.possibleInference ?? null)}`);
@@ -144,8 +147,6 @@ async function main() {
console.log(`answerMeaning.resolutionGuidance: ${JSON.stringify(am.resolutionGuidance ?? null)}`);
}
// ── Capture accepted structural mutation fields ────────
const proposal = updateResult.json.updatedProposal ?? updateResult.json.proposal ?? null;
if (proposal) {
console.log(`updatedNodes: ${JSON.stringify(proposal.updatedNodes ?? [])}`);
console.log(`resolvedUnknownNodeIds: ${JSON.stringify(proposal.resolvedUnknownNodeIds ?? [])}`);
@@ -154,7 +155,7 @@ async function main() {
}
// ── Capture structuralActionRequired from accepted update ─
const sar = updateResult.json.structuralActionRequired;
const sar = proposal?.structuralActionRequired;
if (sar === undefined || sar === null) {
console.log(`structuralActionRequired: null`);
} else {
tests/reproduce-multi-turn-investigation.harness.test.js
+46 -38
View File
@@ -272,13 +272,7 @@ describe("reproduce-multi-turn-investigation harness: one-shot semantics", () =>
stage: "update_applied",
updatedSituationGraph: { nodes: [], edges: [] },
selectedQuestion: { question: "q2" },
answerMeaning: {
userSupportedMeaning: "cost reduction is a primary driver",
possibleInference: "if cost savings not realized, relocation weakens",
supportCategory: "other",
resolutionGuidance: "determine actual projected figures",
},
updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] },
updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [], answerMeaning: { userSupportedMeaning: "cost reduction is a primary driver", possibleInference: "if cost savings not realized, relocation weakens", supportCategory: "other", resolutionGuidance: "determine actual projected figures" } },
}),
});
expect(r.updateCalls).toBe(1);
@@ -389,8 +383,7 @@ describe("reproduce-multi-turn-investigation harness: one-shot semantics", () =>
stage: "update_applied",
updatedSituationGraph: { nodes: [], edges: [] },
selectedQuestion: { question: "q2" },
structuralActionRequired: true,
updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] },
updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [], structuralActionRequired: true },
}),
});
expect(r.startCalls).toBe(1);
@@ -408,8 +401,7 @@ describe("reproduce-multi-turn-investigation harness: one-shot semantics", () =>
stage: "update_applied",
updatedSituationGraph: { nodes: [], edges: [] },
selectedQuestion: { question: "q2" },
structuralActionRequired: false,
updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] },
updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [], structuralActionRequired: false },
}),
});
expect(r.updateCalls).toBe(1);
@@ -444,8 +436,7 @@ describe("reproduce-multi-turn-investigation harness: one-shot semantics", () =>
stage: "update_applied",
updatedSituationGraph: { nodes: [], edges: [] },
selectedQuestion: { question: "q2" },
structuralActionRequired: null,
updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] },
updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [], structuralActionRequired: null },
}),
});
expect(r.updateCalls).toBe(1);
@@ -561,8 +552,7 @@ describe("reproduce-multi-turn-investigation harness: one-shot semantics", () =>
stage: "update_applied",
updatedSituationGraph: { nodes: [], edges: [] },
selectedQuestion: { question: "q" },
structuralActionRequired: true,
updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] },
updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [], structuralActionRequired: true },
}),
});
expect(rSuccess.updateCalls).toBe(2);
@@ -863,6 +853,11 @@ describe("reproduce-multi-turn-investigation harness: one-shot semantics", () =>
edges: [{ fromNodeId: "n_proj_validation", toNodeId: "n_relocation_state", relationship: "depends_on" }],
},
selectedQuestion: { question: "What evidence would clarify validation?", nodeId: "n_proj_validation" },
updatedProposal: {
addedNodes: [{ id: "n_proj_validation" }],
addedEdges: [],
updatedNodes: [],
resolvedUnknownNodeIds: [],
structuralActionRequired: true,
answerMeaning: {
userSupportedMeaning: "User is unsure whether projected savings are realistic.",
@@ -870,11 +865,6 @@ describe("reproduce-multi-turn-investigation harness: one-shot semantics", () =>
supportCategory: "uncertain",
resolutionGuidance: null,
},
updatedProposal: {
addedNodes: [{ id: "n_proj_validation" }],
addedEdges: [],
updatedNodes: [],
resolvedUnknownNodeIds: [],
},
}),
});
@@ -1141,6 +1131,12 @@ function runPreAnchoredSimulation(cfg) {
stage: "update_applied",
updatedSituationGraph: graph,
selectedQuestion: { question: "What evidence would clarify projected savings realism?", nodeId: "n_savings_realism" },
// structuralActionRequired and answerMeaning only inside updatedProposal (production path)
updatedProposal: {
addedNodes: C.updateBehavior === "no-op" ? [] : [{ id: "n_new_unknown", kind: "unknown", label: "test node", description: "test", confidence: "low", value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], parentId: null, childIds: [] }],
addedEdges: C.updateBehavior === "no-op" ? [] : [{ id: "e-new", fromNodeId: "n_new_unknown", toNodeId: "n_relocation_state", relationship: "depends_on", confidence: "medium", description: "test edge" }],
updatedNodes: updatedNodes,
resolvedUnknownNodeIds: [],
structuralActionRequired: C.updateBehavior === "no-op" ? false : true,
answerMeaning: {
userSupportedMeaning: "User is unsure whether the projected office savings from the relocation are realistic.",
@@ -1148,11 +1144,6 @@ function runPreAnchoredSimulation(cfg) {
supportCategory: "uncertain",
resolutionGuidance: null,
},
updatedProposal: {
addedNodes: C.updateBehavior === "no-op" ? [] : [{ id: "n_new_unknown", kind: "unknown", label: "test node", description: "test", confidence: "low", value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], parentId: null, childIds: [] }],
addedEdges: C.updateBehavior === "no-op" ? [] : [{ id: "e-new", fromNodeId: "n_new_unknown", toNodeId: "n_relocation_state", relationship: "depends_on", confidence: "medium", description: "test edge" }],
updatedNodes: updatedNodes,
resolvedUnknownNodeIds: [],
},
}),
};
@@ -1191,10 +1182,10 @@ function runPreAnchoredSimulation(cfg) {
};
}
// Capture fields (mirrors harness print logic)
const am = uj.answerMeaning ?? null;
// Capture fields — mirrors harness production path (from proposal, not root)
let proposal = uj.updatedProposal ?? uj.proposal ?? null;
let sar = uj.structuralActionRequired;
const am = proposal?.answerMeaning ?? null;
let sar = proposal?.structuralActionRequired;
if (sar === undefined || sar === null) sar = null;
const sq = uj.selectedQuestion ?? null;
@@ -1238,10 +1229,26 @@ function runSimulationWithResponseShape(cfg) {
: null;
if (resp) {
return { status: resp.success ? 200 : 422, json: () => resp };
// Production exposes these inside updatedProposal; read from there.
const proposalFromResp = resp.updatedProposal ?? {};
const sar = proposalFromResp.structuralActionRequired !== undefined
? proposalFromResp.structuralActionRequired
: (resp.structuralActionRequired !== undefined ? resp.structuralActionRequired : undefined);
const am = proposalFromResp.answerMeaning ?? resp.answerMeaning ?? null;
return { status: resp.success ? 200 : 422, json: () => ({
...resp,
updatedProposal: {
addedNodes: proposalFromResp.addedNodes ?? [],
addedEdges: proposalFromResp.addedEdges ?? [],
updatedNodes: proposalFromResp.updatedNodes ?? [],
resolvedUnknownNodeIds: proposalFromResp.resolvedUnknownNodeIds ?? [],
structuralActionRequired: sar,
answerMeaning: am,
},
}) };
}
// Fallback to default mock
// Fallback to default mock (no structuralActionRequired or answerMeaning at any level)
const shouldReject = typeof body.answer === "string" && body.answer.includes("_reject");
return {
status: shouldReject ? 422 : 200,
@@ -1281,20 +1288,21 @@ function runSimulationWithResponseShape(cfg) {
graph = uj.updatedSituationGraph;
question = uj.selectedQuestion ? uj.selectedQuestion.question : null;
// Mirror the harness capture (what would be printed)
const am = uj.answerMeaning ?? null;
if (am) captured.answerMeaning = { userSupportedMeaning: am.userSupportedMeaning, possibleInference: am.possibleInference, supportCategory: am.supportCategory, resolutionGuidance: am.resolutionGuidance };
// Mirror the harness capture — reads from proposal (production path), NOT root.
const proposal = uj.updatedProposal ?? uj.proposal ?? null;
if (proposal) captured.proposal = { addedNodes: proposal.addedNodes ?? [], addedEdges: proposal.addedEdges ?? [], updatedNodes: proposal.updatedNodes ?? [], resolvedUnknownNodeIds: proposal.resolvedUnknownNodeIds ?? [] };
// Mirror the harness structuralActionRequired capture for accepted updates
const sar = uj.structuralActionRequired;
if (proposal) {
captured.proposal = { addedNodes: proposal.addedNodes ?? [], addedEdges: proposal.addedEdges ?? [], updatedNodes: proposal.updatedNodes ?? [], resolvedUnknownNodeIds: proposal.resolvedUnknownNodeIds ?? [] };
// Structural fields live inside graphUpdate schema, not at root level.
const sar = proposal.structuralActionRequired;
if (sar === undefined || sar === null) {
captured.structuralActionRequired = null;
} else {
captured.structuralActionRequired = sar;
}
}
const am = proposal?.answerMeaning ?? null;
if (am) captured.answerMeaning = { userSupportedMeaning: am.userSupportedMeaning, possibleInference: am.possibleInference, supportCategory: am.supportCategory, resolutionGuidance: am.resolutionGuidance };
const sq = uj.selectedQuestion ?? null;
if (sq && typeof sq === "object") captured.selectedQuestion = { question: sq.question, nodeId: sq.nodeId };