tooling: add pre-anchored update fixture
This commit is contained in:
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"description": "Deterministic pre-existing graph fixture for false/no-op update testing.",
|
||||
"scenario": "We are considering relocating the engineering team to reduce operating costs.",
|
||||
"unresolved_question": "Are the projected office savings from relocation realistic?",
|
||||
"graph": {
|
||||
"centralStatement": "We are considering relocating the engineering team to reduce operating costs.",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "n_relocation_state",
|
||||
"label": "Engineering team relocation consideration",
|
||||
"description": "Current state: organisation is considering relocating its engineering team from London to Manchester to reduce operating costs.",
|
||||
"kind": "state",
|
||||
"status": "provisional",
|
||||
"confidence": "high",
|
||||
"value": null,
|
||||
"unit": null,
|
||||
"evidenceIds": [],
|
||||
"dependsOn": [],
|
||||
"affects": [],
|
||||
"parentId": null,
|
||||
"childIds": []
|
||||
},
|
||||
{
|
||||
"id": "n_savings_realism",
|
||||
"label": "Are the projected office savings from relocation realistic?",
|
||||
"description": "Uncertainty: whether the projected office savings from the relocation are realistic.",
|
||||
"kind": "unknown",
|
||||
"status": "unknown",
|
||||
"confidence": "low",
|
||||
"value": null,
|
||||
"unit": null,
|
||||
"evidenceIds": [],
|
||||
"dependsOn": ["n_relocation_state"],
|
||||
"affects": [],
|
||||
"parentId": null,
|
||||
"childIds": []
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{
|
||||
"id": "e-sr-to-state",
|
||||
"fromNodeId": "n_savings_realism",
|
||||
"toNodeId": "n_relocation_state",
|
||||
"relationship": "depends_on",
|
||||
"confidence": "medium",
|
||||
"description": "savings-realism uncertainty depends on the relocation consideration state"
|
||||
}
|
||||
],
|
||||
"activeUnknownNodeId": "n_savings_realism",
|
||||
"resolvedNodeIds": [],
|
||||
"currentSummary": "Engineering team is being considered for relocation; savings realism uncertain.",
|
||||
"reasoningState": {
|
||||
"comparabilityStatus": null,
|
||||
"relationshipStatus": null,
|
||||
"relationshipAssessed": false,
|
||||
"contradictionReasoningAllowed": true,
|
||||
"reasoningStages": []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -569,8 +569,433 @@ describe("reproduce-multi-turn-investigation harness: one-shot semantics", () =>
|
||||
expect(rSuccess.startCalls).toBe(1);
|
||||
expect(rSuccess.apiLog.length).toBe(3); // 1 start + 2 updates, no extras
|
||||
});
|
||||
|
||||
// ── 57J.74: pre-anchored update-only fixture tests ───────────────────────
|
||||
|
||||
it("pre-anchored fixture contains exactly one unresolved savings-realism anchor", () => {
|
||||
const graph = PRE_ANCHORED_FIXTURE.graph;
|
||||
const savingsNodes = graph.nodes.filter((n) => n.kind === "unknown" && n.status === "unknown" && n.label.includes("savings"));
|
||||
expect(savingsNodes.length).toBe(1);
|
||||
expect(savingsNodes[0].id).toBe("n_savings_realism");
|
||||
});
|
||||
|
||||
it("pre-anchored fixture uses valid existing graph schema", () => {
|
||||
const g = PRE_ANCHORED_FIXTURE.graph;
|
||||
|
||||
// Graph-level fields
|
||||
expect(typeof g.centralStatement).toBe("string");
|
||||
expect(g.centralStatement.length).toBeGreaterThan(0);
|
||||
expect(Array.isArray(g.nodes)).toBe(true);
|
||||
expect(g.nodes.length).toBeGreaterThanOrEqual(1);
|
||||
expect(Array.isArray(g.edges)).toBe(true);
|
||||
expect(typeof g.currentSummary).toBe("string");
|
||||
expect(g.currentSummary.length).toBeGreaterThan(0);
|
||||
expect(g.activeUnknownNodeId).not.toBeNull();
|
||||
|
||||
// Node fields — each node must have valid kind/status/confidence
|
||||
const validKinds = ["observation", "reported_claim", "metric", "state", "transition", "relationship", "assumption", "unknown", "conclusion"];
|
||||
const validStatuses = ["known", "unknown", "provisional", "supported", "weakened", "contradicted", "resolved"];
|
||||
const validConfidences = ["low", "medium", "high"];
|
||||
|
||||
for (const node of g.nodes) {
|
||||
expect(validKinds).toContain(node.kind);
|
||||
expect(validStatuses).toContain(node.status);
|
||||
expect(validConfidences).toContain(node.confidence);
|
||||
expect(typeof node.id).toBe("string");
|
||||
expect(node.id.length).toBeGreaterThan(0);
|
||||
expect(typeof node.label).toBe("string");
|
||||
expect(node.label.length).toBeGreaterThan(0);
|
||||
expect(typeof node.description).toBe("string");
|
||||
expect(node.description.length).toBeGreaterThan(0);
|
||||
}
|
||||
|
||||
// Edge fields
|
||||
for (const edge of g.edges) {
|
||||
const validRels = ["supports", "weakens", "contradicts", "depends_on", "causes", "may_cause", "measures", "compares_with", "updates", "other"];
|
||||
expect(validRels).toContain(edge.relationship);
|
||||
expect(typeof edge.id).toBe("string");
|
||||
expect(edge.id.length).toBeGreaterThan(0);
|
||||
expect(typeof edge.fromNodeId).toBe("string");
|
||||
expect(edge.fromNodeId.length).toBeGreaterThan(0);
|
||||
expect(typeof edge.toNodeId).toBe("string");
|
||||
expect(edge.toNodeId.length).toBeGreaterThan(0);
|
||||
expect(typeof edge.description).toBe("string");
|
||||
expect(edge.description.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it("pre-anchored fixture contains a valid relationship into the graph", () => {
|
||||
const g = PRE_ANCHORED_FIXTURE.graph;
|
||||
const srNode = g.nodes.find((n) => n.id === "n_savings_realism");
|
||||
expect(srNode).toBeDefined();
|
||||
|
||||
const edgesFromSr = g.edges.filter((e) => e.fromNodeId === srNode.id);
|
||||
expect(edgesFromSr.length).toBeGreaterThan(0);
|
||||
|
||||
const edge = edgesFromSr[0];
|
||||
// The edge connects the unknown into the state node
|
||||
expect(edge.relationship).toBe("depends_on");
|
||||
|
||||
// Both nodes referenced by the edge must exist in the graph
|
||||
const fromNode = g.nodes.find((n) => n.id === edge.fromNodeId);
|
||||
const toNode = g.nodes.find((n) => n.id === edge.toNodeId);
|
||||
expect(fromNode).toBeDefined();
|
||||
expect(toNode).toBeDefined();
|
||||
|
||||
// The edge must reference the savings-realism node as from
|
||||
expect(fromNode.id).toBe("n_savings_realism");
|
||||
});
|
||||
|
||||
it("pre-anchored update-only mode sends the exact fixture graph into the real update request shape", () => {
|
||||
const r = runPreAnchoredSimulation({ answer: "I am unsure whether the projected office savings from the relocation are realistic." });
|
||||
expect(r.startCalls).toBe(0);
|
||||
expect(r.updateCalls).toBe(1);
|
||||
|
||||
// Verify the captured graph matches what was sent
|
||||
const { nodes, edges } = r;
|
||||
const fixtureGraph = PRE_ANCHORED_FIXTURE.graph;
|
||||
expect(nodes.length).toBe(fixtureGraph.nodes.length);
|
||||
expect(edges.length).toBe(fixtureGraph.edges.length);
|
||||
expect(nodes[0].id).toBe(fixtureGraph.nodes[0].id);
|
||||
expect(nodes[1].id).toBe(fixtureGraph.nodes[1].id);
|
||||
});
|
||||
|
||||
it("pre-anchored update-only mode does not call Start", () => {
|
||||
const r = runPreAnchoredSimulation();
|
||||
expect(r.startCalls).toBe(0);
|
||||
expect(r.updateCalls).toBe(1);
|
||||
|
||||
const startEntries = r.apiLog.filter((e) => e.step === "start");
|
||||
const updateEntries = r.apiLog.filter((e) => e.step === "update");
|
||||
expect(startEntries.length).toBe(0);
|
||||
expect(updateEntries.length).toBe(1);
|
||||
});
|
||||
|
||||
it("pre-anchored update-only mode makes exactly one Update call", () => {
|
||||
const r = runPreAnchoredSimulation();
|
||||
expect(r.updateCalls).toBe(1);
|
||||
expect(r.type).toBe("all_success");
|
||||
|
||||
const updateEntries = r.apiLog.filter((e) => e.step === "update");
|
||||
expect(updateEntries.length).toBe(1);
|
||||
});
|
||||
|
||||
it("normal Start→Update harness mode remains unchanged", () => {
|
||||
// Verify existing normal mode still works with exactly 1 start call
|
||||
const rNormal = runSimulation({ scenario: "test_ok", maxUpdates: 1, answers: ["good answer"] });
|
||||
expect(rNormal.startCalls).toBe(1);
|
||||
expect(rNormal.updateCalls).toBeGreaterThanOrEqual(0);
|
||||
|
||||
// Normal mode should not be affected by the new pre-anchored code paths
|
||||
const rWithShape = runSimulationWithResponseShape({
|
||||
scenario: "test_ok",
|
||||
maxUpdates: 1,
|
||||
answers: ["good answer"],
|
||||
onResponseUpdate: () => ({
|
||||
success: true,
|
||||
stage: "update_applied",
|
||||
updatedSituationGraph: { nodes: [], edges: [] },
|
||||
selectedQuestion: { question: "q2" },
|
||||
updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] },
|
||||
}),
|
||||
});
|
||||
expect(rWithShape.startCalls).toBe(1);
|
||||
expect(rWithShape.updateCalls).toBe(1);
|
||||
});
|
||||
|
||||
it("57J.62 accepted/rejected capture hardening remains unchanged", () => {
|
||||
// Capture addedNodes via accepted response shape (existing test pattern)
|
||||
const rAccepted = runSimulationWithResponseShape({
|
||||
scenario: "test_ok",
|
||||
maxUpdates: 1,
|
||||
answers: ["good answer"],
|
||||
onResponseUpdate: () => ({
|
||||
success: true,
|
||||
stage: "update_applied",
|
||||
updatedSituationGraph: { nodes: [], edges: [] },
|
||||
selectedQuestion: { question: "q2" },
|
||||
updatedProposal: {
|
||||
addedNodes: [{ id: "n_test", kind: "unknown", label: "test", description: "test", confidence: "low", value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], parentId: null, childIds: [] }],
|
||||
addedEdges: [],
|
||||
updatedNodes: [{ nodeId: "n_existing", newValue: "confirmed" }],
|
||||
resolvedUnknownNodeIds: ["n_resolved"],
|
||||
},
|
||||
}),
|
||||
});
|
||||
expect(rAccepted.captured.proposal.addedNodes.length).toBe(1);
|
||||
expect(rAccepted.captured.proposal.updatedNodes.length).toBe(1);
|
||||
expect(rAccepted.captured.proposal.resolvedUnknownNodeIds.length).toBe(1);
|
||||
|
||||
// Capture rejected snapshot via rejection (existing test pattern)
|
||||
const rRejected = runSimulation({ scenario: "test_ok", maxUpdates: 1, answers: ["answer_reject"] });
|
||||
expect(rRejected.type).toBe("update_rejection");
|
||||
expect(rRejected.startCalls).toBe(1);
|
||||
expect(rRejected.updateCalls).toBe(1);
|
||||
});
|
||||
|
||||
it("57J.72 direct structuralActionRequired capture remains unchanged", () => {
|
||||
// Accepted update with true (existing test pattern)
|
||||
const rTrue = runSimulationWithResponseShape({
|
||||
scenario: "test_ok",
|
||||
maxUpdates: 1,
|
||||
answers: ["good answer"],
|
||||
onResponseUpdate: () => ({
|
||||
success: true,
|
||||
stage: "update_applied",
|
||||
updatedSituationGraph: { nodes: [], edges: [] },
|
||||
selectedQuestion: { question: "q2" },
|
||||
structuralActionRequired: true,
|
||||
updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] },
|
||||
}),
|
||||
});
|
||||
expect(rTrue.captured.structuralActionRequired).toBe(true);
|
||||
|
||||
// Accepted update with false (existing test pattern)
|
||||
const rFalse = runSimulationWithResponseShape({
|
||||
scenario: "test_ok",
|
||||
maxUpdates: 1,
|
||||
answers: ["good answer"],
|
||||
onResponseUpdate: () => ({
|
||||
success: true,
|
||||
stage: "update_applied",
|
||||
updatedSituationGraph: { nodes: [], edges: [] },
|
||||
selectedQuestion: { question: "q2" },
|
||||
structuralActionRequired: false,
|
||||
updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] },
|
||||
}),
|
||||
});
|
||||
expect(rFalse.captured.structuralActionRequired).toBe(false);
|
||||
|
||||
// Absent field reports null (existing test pattern)
|
||||
const rAbsent = runSimulationWithResponseShape({
|
||||
scenario: "test_ok",
|
||||
maxUpdates: 1,
|
||||
answers: ["good answer"],
|
||||
onResponseUpdate: () => ({
|
||||
success: true,
|
||||
stage: "update_applied",
|
||||
updatedSituationGraph: { nodes: [], edges: [] },
|
||||
selectedQuestion: { question: "q2" },
|
||||
updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] },
|
||||
}),
|
||||
});
|
||||
expect(rAbsent.captured.structuralActionRequired).toBeNull();
|
||||
|
||||
// Rejected snapshot (existing test pattern)
|
||||
const rSnapshot = runSimulationWithRejectionSnapshot({
|
||||
scenario: "test_ok",
|
||||
maxUpdates: 1,
|
||||
answers: ["answer_reject"],
|
||||
rejectedSnapshot: { structuralActionRequired: true },
|
||||
});
|
||||
expect(rSnapshot.rejectedSnapshot.structuralActionRequired).toBe(true);
|
||||
});
|
||||
|
||||
it("no retries or supplementary calls are introduced in pre-anchored mode", () => {
|
||||
const r = runPreAnchoredSimulation();
|
||||
const totalCalls = r.startCalls + r.updateCalls;
|
||||
expect(totalCalls).toBe(1); // 0 start + 1 update only
|
||||
expect(r.apiLog.length).toBe(1);
|
||||
expect(r.type).toBe("all_success");
|
||||
expect(r.exitCode).toBe(0);
|
||||
|
||||
const retryEntries = r.apiLog.filter((e) => e.step === "update" && e.answer?.includes("_retry"));
|
||||
expect(retryEntries.length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Pre-anchored update-only mode (57J.74) ─────────────
|
||||
|
||||
/**
|
||||
* Deterministic fixture used by pre-anchored tests.
|
||||
* Scenario: considering relocating engineering team; savings-realism unknown already present.
|
||||
*/
|
||||
const PRE_ANCHORED_FIXTURE = {
|
||||
description: "Deterministic pre-existing graph fixture for false/no-op update testing.",
|
||||
scenario: "We are considering relocating the engineering team to reduce operating costs.",
|
||||
unresolvedQuestion: "Are the projected office savings from relocation realistic?",
|
||||
graph: {
|
||||
centralStatement:
|
||||
"We are considering relocating the engineering team to reduce operating costs.",
|
||||
nodes: [
|
||||
{
|
||||
id: "n_relocation_state",
|
||||
label: "Engineering team relocation consideration",
|
||||
description:
|
||||
"Current state: organisation is considering relocating its engineering team from London to Manchester to reduce operating costs.",
|
||||
kind: "state",
|
||||
status: "provisional",
|
||||
confidence: "high",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
},
|
||||
{
|
||||
id: "n_savings_realism",
|
||||
label: "Are the projected office savings from relocation realistic?",
|
||||
description:
|
||||
"Uncertainty: whether the projected office savings from the relocation are realistic.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "low",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: ["n_relocation_state"],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
},
|
||||
],
|
||||
edges: [
|
||||
{
|
||||
id: "e-sr-to-state",
|
||||
fromNodeId: "n_savings_realism",
|
||||
toNodeId: "n_relocation_state",
|
||||
relationship: "depends_on",
|
||||
confidence: "medium",
|
||||
description:
|
||||
"savings-realism uncertainty depends on the relocation consideration state",
|
||||
},
|
||||
],
|
||||
activeUnknownNodeId: "n_savings_realism",
|
||||
resolvedNodeIds: [],
|
||||
currentSummary:
|
||||
"Engineering team is being considered for relocation; savings realism uncertain.",
|
||||
reasoningState: {
|
||||
comparabilityStatus: null,
|
||||
relationshipStatus: null,
|
||||
relationshipAssessed: false,
|
||||
contradictionReasoningAllowed: true,
|
||||
reasoningStages: [],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Synchronous simulator of the pre-anchored update-only harness mode.
|
||||
* Mirrors what reproduce-multi-turn-investigation.mjs does when fixtureMode is "updateOnly".
|
||||
*/
|
||||
function runPreAnchoredSimulation(cfg) {
|
||||
let startCalls = 0;
|
||||
let updateCalls = 0;
|
||||
let apiLog = [];
|
||||
|
||||
const initialGraph = cfg?.initialGraph || PRE_ANCHORED_FIXTURE.graph;
|
||||
const answer = cfg?.answer || "I am unsure whether the projected office savings from the relocation are realistic.";
|
||||
|
||||
// Normalise cfg to safe defaults so the closure always reads non-null fields.
|
||||
const C = { ...cfg, initialGraph: undefined, answer: undefined };
|
||||
|
||||
// Pre-anchored: no Start call — graph is supplied directly.
|
||||
let graph = JSON.parse(JSON.stringify(initialGraph));
|
||||
let question = null; // will be set by the update response (or mock)
|
||||
|
||||
const api = {
|
||||
post(path, body) {
|
||||
if (path === "/api/cases/start") {
|
||||
apiLog.push({ step: "start" });
|
||||
startCalls++;
|
||||
return { status: 200, json: () => ({ success: true, situationGraph: { nodes: [], edges: [] }, selectedQuestion: { question: "q" } }) };
|
||||
}
|
||||
|
||||
if (path === "/api/cases/update") {
|
||||
apiLog.push({ step: "update", answer: body.answer });
|
||||
updateCalls++;
|
||||
|
||||
const resp = typeof C.onResponseUpdate === "function"
|
||||
? C.onResponseUpdate(updateCalls - 1)
|
||||
: null;
|
||||
|
||||
if (resp) {
|
||||
return { status: resp.success ? 200 : 422, json: () => resp };
|
||||
}
|
||||
|
||||
// Default pre-anchored success response
|
||||
const updatedNodes = C.updateBehavior === "no-op"
|
||||
? [{ nodeId: graph.nodes[1].id, newValue: null }]
|
||||
: [];
|
||||
|
||||
return {
|
||||
status: 200,
|
||||
json: () => ({
|
||||
success: true,
|
||||
stage: "update_applied",
|
||||
updatedSituationGraph: graph,
|
||||
selectedQuestion: { question: "What evidence would clarify projected savings realism?", nodeId: "n_savings_realism" },
|
||||
structuralActionRequired: C.updateBehavior === "no-op" ? false : true,
|
||||
answerMeaning: {
|
||||
userSupportedMeaning: "User is unsure whether the projected office savings from the relocation are realistic.",
|
||||
possibleInference: null,
|
||||
supportCategory: "uncertain",
|
||||
resolutionGuidance: null,
|
||||
},
|
||||
updatedProposal: {
|
||||
addedNodes: C.updateBehavior === "no-op" ? [] : [{ id: "n_new_unknown", kind: "unknown", label: "test node", description: "test", confidence: "low", value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], parentId: null, childIds: [] }],
|
||||
addedEdges: C.updateBehavior === "no-op" ? [] : [{ id: "e-new", fromNodeId: "n_new_unknown", toNodeId: "n_relocation_state", relationship: "depends_on", confidence: "medium", description: "test edge" }],
|
||||
updatedNodes: updatedNodes,
|
||||
resolvedUnknownNodeIds: [],
|
||||
},
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
apiLog.push({ step: "unknown", path });
|
||||
return { status: 404, json: () => ({ error: "not found" }) };
|
||||
},
|
||||
};
|
||||
|
||||
// ── Pre-anchored: no Start call. Use supplied graph directly. ──
|
||||
|
||||
// Verify fixture integrity before proceeding (what harness would report)
|
||||
const nodes = initialGraph.nodes;
|
||||
const edges = initialGraph.edges;
|
||||
const savingsNodes = nodes.filter((n) => n.kind === "unknown" && n.status === "unknown" && n.label.includes("savings"));
|
||||
|
||||
if (cfg?.requireAnchor !== false && savingsNodes.length !== 1) {
|
||||
return {
|
||||
startCalls, updateCalls, type: "anchor_validation_failed", exitCode: 1,
|
||||
anchorCount: savingsNodes.length, apiLog, nodes, edges,
|
||||
};
|
||||
}
|
||||
|
||||
// Pre-anchored mode sends the exact fixture graph into the update request.
|
||||
const upResp = api.post("/api/cases/update", {
|
||||
situationGraph: graph,
|
||||
previousQuestion: question,
|
||||
answer,
|
||||
});
|
||||
const uj = upResp.json();
|
||||
|
||||
if (!uj.success) {
|
||||
return {
|
||||
startCalls, updateCalls, type: "update_rejection", exitCode: 1, apiLog, nodes, edges,
|
||||
};
|
||||
}
|
||||
|
||||
// Capture fields (mirrors harness print logic)
|
||||
const am = uj.answerMeaning ?? null;
|
||||
let proposal = uj.updatedProposal ?? uj.proposal ?? null;
|
||||
let sar = uj.structuralActionRequired;
|
||||
if (sar === undefined || sar === null) sar = null;
|
||||
const sq = uj.selectedQuestion ?? null;
|
||||
|
||||
return {
|
||||
startCalls, updateCalls, type: "all_success", exitCode: 0, apiLog, nodes, edges,
|
||||
captured: {
|
||||
answerMeaning: am ? { userSupportedMeaning: am.userSupportedMeaning, possibleInference: am.possibleInference, supportCategory: am.supportCategory, resolutionGuidance: am.resolutionGuidance } : null,
|
||||
proposal: proposal ? { addedNodes: proposal.addedNodes ?? [], addedEdges: proposal.addedEdges ?? [], updatedNodes: proposal.updatedNodes ?? [], resolvedUnknownNodeIds: proposal.resolvedUnknownNodeIds ?? [] } : null,
|
||||
structuralActionRequired: sar,
|
||||
selectedQuestion: sq && typeof sq === "object" ? { question: sq.question, nodeId: sq.nodeId } : null,
|
||||
graph: { nodes: uj.updatedSituationGraph?.nodes ?? [], edges: uj.updatedSituationGraph?.edges ?? [] },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extended simulation with capture recording — mirrors what the updated harness prints.
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user