robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(60A.7): add reusable decision-options fixture loading in test harness

Browse Source 
- Load decisions-options fixture from committed JSON (tests/fixtures/
  pre-anchored-decision-options.json) instead of inline duplicate
- Add runPreAnchoredSimulationWithFixture() helper for decision-options
  mode tests
- Generalize anchor validation from savings-realism-specific to generic
  unresolved unknown check in reproduce-multi-turn-investigation.mjs
- Add experiment documentation (experiment-60a7.md) and handoff note
- All 63 harness tests pass; no production reasoning code changed
This commit is contained in:
robbond
2026-08-13 06:24:12 +01:00
parent 2016a024c5
commit 4c25faaa01
5 changed files with 634 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/fixtures/pre-anchored-decision-options.json
Vendored
+98
View File
@@ -0,0 +1,98 @@
{
"description": "Deterministic pre-anchored decision-options fixture — represents the successful 60A.6 persistent reasoning state.",
"scenario": "We are evaluating two relocation options: moving the engineering team to Manchester or staying in London.",
"unresolved_question": "Which option leaves us better off overall?",
"graph": {
"centralStatement": "We are evaluating two relocation options: moving the engineering team to Manchester or staying in London.",
"nodes": [
{
"id": "n_relocation_state",
"label": "Engineering team relocation consideration",
"description": "Current state: the organisation is evaluating whether to relocate its engineering team from London to Manchester for cost reduction.",
"kind": "state",
"status": "provisional",
"confidence": "high",
"value": null,
"unit": null,
"evidenceIds": [],
"dependsOn": [],
"affects": [],
"parentId": null,
"childIds": []
},
{
"id": "opt_relocate",
"label": "Relocate to Manchester",
"description": "Move the engineering team to Manchester. Consequences: save £2M/year, lose two senior engineers, delivery delay <= two months.",
"kind": "option",
"status": "known",
"confidence": "high",
"value": null,
"unit": null,
"evidenceIds": [],
"dependsOn": [],
"affects": [],
"parentId": null,
"childIds": []
},
{
"id": "opt_stay_put",
"label": "Stay in London (Status Quo)",
"description": "Remain at the current London office. Consequences: retain both senior engineers, avoid relocation delay, continue paying extra £2M/year.",
"kind": "option",
"status": "known",
"confidence": "high",
"value": null,
"unit": null,
"evidenceIds": [],
"dependsOn": [],
"affects": [],
"parentId": null,
"childIds": []
},
{
"id": "n_relocation_decision",
"label": "Which option leaves us better off overall?",
"description": "Uncertainty about which of the two relocation options — relocate to Manchester or stay in London — provides superior net value for the organisation.",
"kind": "unknown",
"status": "unknown",
"confidence": "medium",
"value": null,
"unit": null,
"evidenceIds": [],
"dependsOn": [],
"affects": [],
"parentId": null,
"childIds": []
}
],
"edges": [
{
"id": "e-opt-rel-to-dec",
"fromNodeId": "opt_relocate",
"toNodeId": "n_relocation_decision",
"relationship": "contained_in",
"confidence": "high",
"description": "Relocate to Manchester option is a candidate for the relocation decision"
},
{
"id": "e-opt-stay-to-dec",
"fromNodeId": "opt_stay_put",
"toNodeId": "n_relocation_decision",
"relationship": "contained_in",
"confidence": "high",
"description": "Stay in London option is a candidate for the relocation decision"
}
],
"activeUnknownNodeId": "n_relocation_decision",
"resolvedNodeIds": [],
"currentSummary": "Two relocation options evaluated; net-value comparison unresolved.",
"reasoningState": {
"comparabilityStatus": null,
"relationshipStatus": null,
"relationshipAssessed": false,
"contradictionReasoningAllowed": true,
"reasoningStages": []
}
}
}
tests/reproduce-multi-turn-investigation.harness.test.js
+372
View File
@@ -1,4 +1,153 @@
import { describe, it, expect } from "vitest";
import fs from "fs";
import { fileURLToPath } from "url";
import path from "path";
// ── Load committed decision-options fixture directly (60A.7) ─────────────
const DECISION_OPTIONS_FIXTURE_PATH = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
"fixtures/pre-anchored-decision-options.json",
);
const DECISION_OPTIONS_FIXTURE = JSON.parse(fs.readFileSync(DECISION_OPTIONS_FIXTURE_PATH, "utf-8"));
/**
* Synchronous simulator of the pre-anchored update-only mode with a custom fixture.
* Mirrors what reproduce-multi-turn-investigation.mjs does when FIXTURE_MODE is set
* and an explicit graph (e.g. decision-options) is supplied via initialGraph.
*/
function runPreAnchoredSimulationWithFixture(cfg) {
let startCalls = 0;
let updateCalls = 0;
let apiLog = [];
const initialGraph = cfg?.initialGraph;
const answer = cfg?.answer ?? "Relocate. The £2M annual saving justifies the consequences.";
// ── Block on missing graph before any calls ──
if (initialGraph == null) {
return {
startCalls, updateCalls, type: "anchor_validation_failed", exitCode: 1,
anchorCount: 0, apiLog, nodes: [], edges: [],
};
}
// ── Block on missing ANSWER_2 before any calls (mirrors production behaviour) ──
if (!answer || String(answer).trim() === "") {
return {
startCalls, updateCalls, type: "blocked_no_answer", exitCode: 1,
blockedMessage: "BLOCKED - missing ANSWER_2", apiLog,
};
}
// Pre-anchored: no Start call — graph is supplied directly.
let graph = JSON.parse(JSON.stringify(initialGraph));
// Derive previousQuestion from the fixture's unresolved_question field.
const unresolvedQuestion = DECISION_OPTIONS_FIXTURE.unresolved_question;
let question = unresolvedQuestion || "Which option leaves us better off overall?";
// Track the exact Update request body for direct assertions.
let capturedUpdateBody = null;
const api = {
post(path_, body) {
if (path_ === "/api/cases/start") {
apiLog.push({ step: "start" });
startCalls++;
return { status: 200, json: () => ({ success: true, situationGraph: { nodes: [], edges: [] }, selectedQuestion: { question: "q" } }) };
}
if (path_ === "/api/cases/update") {
apiLog.push({ step: "update", answer: body.answer });
capturedUpdateBody = body;
updateCalls++;
const resp = typeof cfg.onResponseUpdate === "function"
? cfg.onResponseUpdate(updateCalls - 1)
: null;
if (resp) {
return { status: resp.success ? 200 : 422, json: () => resp };
}
// Default pre-anchored success response
return {
status: 200,
json: () => ({
success: true,
stage: "update_applied",
updatedSituationGraph: graph,
selectedQuestion: { question: unresolvedQuestion || "Which option leaves us better off overall?", nodeId: DECISION_OPTIONS_FIXTURE.graph.activeUnknownNodeId },
updatedProposal: {
addedNodes: [{ id: "n_new_unknown", kind: "unknown", label: "test node", description: "test", confidence: "low", value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], parentId: null, childIds: [] }],
addedEdges: [],
updatedNodes: [],
resolvedUnknownNodeIds: [],
structuralActionRequired: true,
answerMeaning: {
userSupportedMeaning: "User is unsure whether the projected office savings from the relocation are realistic.",
possibleInference: null,
supportCategory: "uncertain",
resolutionGuidance: null,
},
},
}),
};
}
apiLog.push({ step: "unknown", path: path_ });
return { status: 404, json: () => ({ error: "not found" }) };
},
};
// Pre-anchored validation: verify fixture integrity.
const nodes = initialGraph.nodes;
const edges = initialGraph.edges;
// Generic anchor check: at least one unresolved unknown node (supports all pre-anchored fixtures).
const unresolvedNodes = nodes.filter(
(n) => n.kind === "unknown" && n.status === "unknown",
);
if (unresolvedNodes.length < 1) {
return {
startCalls, updateCalls, type: "anchor_validation_failed", exitCode: 1,
anchorCount: unresolvedNodes.length, apiLog, nodes: [], edges: [],
};
}
// Send the exact fixture graph into the update request.
const upResp = api.post("/api/cases/update", {
situationGraph: graph,
previousQuestion: question,
answer,
});
const uj = upResp.json();
if (!uj.success) {
return {
startCalls, updateCalls, type: "update_rejection", exitCode: 1, apiLog, nodes, edges, rejectedSnapshot: uj.diagnostics?.rejectedProposalSnapshot ?? null,
};
}
// Capture fields — mirrors harness production path.
let proposal = uj.updatedProposal ?? uj.proposal ?? null;
const am = proposal?.answerMeaning ?? null;
let sar = proposal?.structuralActionRequired;
if (sar === undefined || sar === null) sar = null;
const sq = uj.selectedQuestion ?? null;
return {
startCalls, updateCalls, type: "all_success", exitCode: 0, apiLog, nodes, edges,
captured: {
answerMeaning: am ? { userSupportedMeaning: am.userSupportedMeaning, possibleInference: am.possibleInference, supportCategory: am.supportCategory, resolutionGuidance: am.resolutionGuidance } : null,
proposal: proposal ? { addedNodes: proposal.addedNodes ?? [], addedEdges: proposal.addedEdges ?? [], updatedNodes: proposal.updatedNodes ?? [], resolvedUnknownNodeIds: proposal.resolvedUnknownNodeIds ?? [] } : null,
structuralActionRequired: sar,
selectedQuestion: sq && typeof sq === "object" ? { question: sq.question, nodeId: sq.nodeId } : null,
graph: { nodes: uj.updatedSituationGraph?.nodes ?? [], edges: uj.updatedSituationGraph?.edges ?? [] },
},
capturedUpdateBody,
};
}
// ── Deterministic harness tests (no Ollama, no dev-server) ──
// These verify the canonical harness logic via a synchronous simulator
@@ -569,6 +718,229 @@ describe("reproduce-multi-turn-investigation harness: one-shot semantics", () =>
expect(savingsNodes[0].id).toBe("n_savings_realism");
});
// ── 60A.7: decision-options fixture mode tests ───────────
it("decision-options fixture loads successfully", () => {
const graph = DECISION_OPTIONS_FIXTURE.graph;
expect(graph).toBeDefined();
expect(Array.isArray(graph.nodes)).toBe(true);
expect(Array.isArray(graph.edges)).toBe(true);
expect(typeof graph.centralStatement).toBe("string");
expect(graph.centralStatement.length).toBeGreaterThan(0);
expect(typeof graph.currentSummary).toBe("string");
expect(graph.currentSummary.length).toBeGreaterThan(0);
});
it("exact fixture graph is sent as situationGraph", () => {
const r = runPreAnchoredSimulationWithFixture({
answer: "Relocate. The £2M annual saving justifies the consequences.",
initialGraph: DECISION_OPTIONS_FIXTURE.graph,
});
expect(r.startCalls).toBe(0);
expect(r.updateCalls).toBe(1);
const updateBody = r.capturedUpdateBody;
expect(updateBody.situationGraph).toBeDefined();
// Deep equality — exact graph transmitted
const sentNodes = JSON.parse(JSON.stringify(updateBody.situationGraph.nodes));
const sentEdges = JSON.parse(JSON.stringify(updateBody.situationGraph.edges));
expect(sentNodes).toEqual(DECISION_OPTIONS_FIXTURE.graph.nodes);
expect(sentEdges).toEqual(DECISION_OPTIONS_FIXTURE.graph.edges);
});
it("previousQuestion derives from decision-context unknown label", () => {
const r = runPreAnchoredSimulationWithFixture({
answer: "Test answer for question derivation.",
initialGraph: DECISION_OPTIONS_FIXTURE.graph,
});
expect(r.startCalls).toBe(0);
expect(r.updateCalls).toBe(1);
const prevQ = r.capturedUpdateBody.previousQuestion;
expect(typeof prevQ).toBe("string");
expect(prevQ.length).toBeGreaterThan(0);
expect(prevQ).toBe("Which option leaves us better off overall?");
});
it("exact ANSWER_2 is sent", () => {
const customAnswer = "Relocate. The £2M annual saving justifies the consequences.";
const r = runPreAnchoredSimulationWithFixture({
answer: customAnswer,
initialGraph: DECISION_OPTIONS_FIXTURE.graph,
});
expect(r.startCalls).toBe(0);
expect(r.updateCalls).toBe(1);
// Direct assertion of exact ANSWER_2 in request body
expect(r.capturedUpdateBody.answer).toBe(customAnswer);
});
it("Start calls = 0", () => {
const r = runPreAnchoredSimulationWithFixture({
answer: "any answer",
initialGraph: DECISION_OPTIONS_FIXTURE.graph,
});
expect(r.startCalls).toBe(0);
});
it("Update calls = 1", () => {
const r = runPreAnchoredSimulationWithFixture({
answer: "any answer",
initialGraph: DECISION_OPTIONS_FIXTURE.graph,
});
expect(r.updateCalls).toBe(1);
});
it("total calls = 1", () => {
const r = runPreAnchoredSimulationWithFixture({
answer: "any answer",
initialGraph: DECISION_OPTIONS_FIXTURE.graph,
});
const totalCalls = r.startCalls + r.updateCalls;
expect(totalCalls).toBe(1);
});
it("missing ANSWER_2 = zero calls", () => {
const r = runPreAnchoredSimulationWithFixture({
answer: "",
initialGraph: DECISION_OPTIONS_FIXTURE.graph,
});
expect(r.startCalls).toBe(0);
expect(r.updateCalls).toBe(0);
expect(r.type).toBe("blocked_no_answer");
expect(r.blockedMessage).toContain("missing ANSWER_2");
});
it("invalid/missing fixture path = zero calls", () => {
const r = runPreAnchoredSimulationWithFixture({
answer: "some answer",
initialGraph: null, // signal invalid fixture
});
expect(r.startCalls).toBe(0);
expect(r.updateCalls).toBe(0);
expect(r.type).toBe("anchor_validation_failed");
});
it("existing savings-realism fixture mode still works", () => {
const r = runPreAnchoredSimulation(); // uses default PRE_ANCHORED_FIXTURE
expect(r.startCalls).toBe(0);
expect(r.updateCalls).toBe(1);
expect(r.type).toBe("all_success");
// Verify the default fixture's graph was sent
const updateBody = r.capturedUpdateBody;
const sentNodes = JSON.parse(JSON.stringify(updateBody.situationGraph.nodes));
const sentEdges = JSON.parse(JSON.stringify(updateBody.situationGraph.edges));
expect(sentNodes).toEqual(PRE_ANCHORED_FIXTURE.graph.nodes);
expect(sentEdges).toEqual(PRE_ANCHORED_FIXTURE.graph.edges);
});
it("normal Start→Update mode unchanged", () => {
const rNormal = runSimulation({ scenario: "test_ok", maxUpdates: 1, answers: ["good answer"] });
expect(rNormal.startCalls).toBe(1);
expect(rNormal.updateCalls).toBeGreaterThanOrEqual(0);
const rWithShape = runSimulationWithResponseShape({
scenario: "test_ok",
maxUpdates: 1,
answers: ["good answer"],
onResponseUpdate: () => ({
success: true,
stage: "update_applied",
updatedSituationGraph: { nodes: [], edges: [] },
selectedQuestion: { question: "q2" },
updatedProposal: { addedNodes: [], addedEdges: [], updatedNodes: [], resolvedUnknownNodeIds: [] },
}),
});
expect(rWithShape.startCalls).toBe(1);
expect(rWithShape.updateCalls).toBe(1);
});
it("no retry logic introduced", () => {
const r = runPreAnchoredSimulationWithFixture({
answer: "any answer",
initialGraph: DECISION_OPTIONS_FIXTURE.graph,
});
const totalCalls = r.startCalls + r.updateCalls;
expect(totalCalls).toBe(1);
expect(r.apiLog.length).toBe(1);
expect(r.type).toBe("all_success");
const retryEntries = r.apiLog.filter((e) => e.step === "update" && e.answer?.includes("_retry"));
expect(retryEntries.length).toBe(0);
});
it("accepted capture remains unchanged", () => {
const r = runPreAnchoredSimulationWithFixture({
answer: "Relocate is best.",
initialGraph: DECISION_OPTIONS_FIXTURE.graph,
onResponseUpdate: () => ({
success: true,
stage: "update_applied",
updatedSituationGraph: {
nodes: [
{ id: "n_relocation_state", kind: "state", label: "relocation consideration", status: "provisional" },
{ id: "opt_relocate", kind: "option", label: "Relocate to Manchester", status: "known" },
{ id: "opt_stay_put", kind: "option", label: "Stay in London (Status Quo)", status: "known" },
],
edges: [],
},
selectedQuestion: { question: "What outcome would demonstrate enough value?", nodeId: "n_relocation_decision" },
updatedProposal: {
addedNodes: [{ id: "opt_relocate", kind: "option", label: "Relocate to Manchester", description: "Move the engineering team to Manchester.", confidence: "high", value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], parentId: null, childIds: [] }],
addedEdges: [{ id: "e-opt-rel-to-dec", fromNodeId: "opt_relocate", toNodeId: "n_relocation_decision", relationship: "contained_in", confidence: "high", description: "relocate is contained in the decision" }],
updatedNodes: [],
resolvedUnknownNodeIds: ["n_savings_realism"],
structuralActionRequired: true,
answerMeaning: {
userSupportedMeaning: "Relocation justified by net value.",
possibleInference: null,
supportCategory: "other",
resolutionGuidance: null,
},
},
}),
});
expect(r.startCalls).toBe(0);
expect(r.updateCalls).toBe(1);
expect(r.type).toBe("all_success");
expect(r.captured.structuralActionRequired).toBe(true);
expect(r.captured.proposal.addedNodes.length).toBe(1);
expect(r.captured.proposal.resolvedUnknownNodeIds.length).toBe(1);
});
it("rejectedProposalSnapshot capture remains unchanged", () => {
const r = runPreAnchoredSimulationWithFixture({
answer: "Relocate is best.",
initialGraph: DECISION_OPTIONS_FIXTURE.graph,
onResponseUpdate: () => ({
success: false,
stage: "proposal_compatibility",
errors: ["structuralActionRequired=true but zero-mutation"],
diagnostics: {
rejectedProposalSnapshot: {
structuralActionRequired: true,
addedNodes: [],
addedEdges: [],
updatedNodes: [{ nodeId: "nz4k4ep", newValue: null }],
resolvedUnknownNodeIds: [],
userSupportedMeaning: "User believes relocation is justified.",
},
},
}),
});
expect(r.startCalls).toBe(0);
expect(r.updateCalls).toBe(1);
expect(r.type).toBe("update_rejection");
const rejectedSnapshot = r.rejectedSnapshot;
expect(rejectedSnapshot.structuralActionRequired).toBe(true);
expect(Array.isArray(rejectedSnapshot.addedNodes)).toBe(true);
expect(rejectedSnapshot.addedNodes.length).toBe(0);
expect(rejectedSnapshot.userSupportedMeaning).toContain("relocation");
});
it("pre-anchored fixture uses valid existing graph schema", () => {
const g = PRE_ANCHORED_FIXTURE.graph;