experiment: add rejected proposal diagnostics to failure path
Adds rejectedProposalSnapshot to orchestrator diagnostics for proposal_compatibility rejections — exposing answerMeaning (userSupportedMeaning, possibleInference), addedNodes structural fields, addedEdges structural fields, updatedNodes summaries, and resolvedUnknownNodeIds. Diagnostic evidence only; does not alter validation, mutation, or error messages. Stage-gated to proposal_compatibility only.
This commit is contained in:
@@ -3208,4 +3208,61 @@ describe("applyValidatedProposal", () => {
|
||||
}
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
// ── Experiment 57J.31 — diagnostics integration tests (rejecting fixture unchanged) ──
|
||||
|
||||
it("identical rejected fixture still rejects (same stage, no new mutations)", () => {
|
||||
const { graph, proposal } = makeApplicationFixture();
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
proposal: {
|
||||
...proposal,
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: "n-ghost",
|
||||
label: "Ghost unknown",
|
||||
description: "An unknown not grounded in any existing node.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
}),
|
||||
],
|
||||
addedEdges: [
|
||||
makeEdge({
|
||||
id: "e-ghost-edge",
|
||||
fromNodeId: "n-ghost",
|
||||
toNodeId: "neb1bz2",
|
||||
relationship: "depends_on",
|
||||
confidence: "medium",
|
||||
description: "Ghost edge.",
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.stage).toBe("proposal_compatibility");
|
||||
expect(result.updatedSituationGraph).toBeUndefined();
|
||||
expect(result.changesApplied).toBeUndefined();
|
||||
});
|
||||
|
||||
it("successful proposal behaviour unchanged (same pass result, same applied mutations)", () => {
|
||||
const fixture = makeComparabilityUpdateFixture();
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: fixture.graph,
|
||||
proposal: fixture.proposal,
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
// applyValidatedProposal does NOT return a stage field on success — only on failure.
|
||||
// The comparability unknown should be resolved with the answer-provided value.
|
||||
const resolvedNode = result.updatedSituationGraph.nodes.find(
|
||||
(n) => n.id === fixture.comparabilityUnknownId,
|
||||
);
|
||||
expect(resolvedNode.status).toBe("resolved");
|
||||
expect(resolvedNode.value).toBe(
|
||||
"Both figures cover the same accounting period and are taken from the same management accounts.",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mockUpdateCase = vi.fn();
|
||||
|
||||
vi.mock("@/lib/graph/orchestrator.js", () => ({
|
||||
updateCase: (...args) => mockUpdateCase(...args),
|
||||
}));
|
||||
|
||||
// Helper that simulates a proposal_compatibility rejection (used in some tests)
|
||||
function createCompetitionRejection() {
|
||||
return {
|
||||
success: false,
|
||||
stage: "proposal_compatibility",
|
||||
errors: [
|
||||
"answerMeaning.userSupportedMeaning introduces a stronger reasoning category than the raw answer establishes.",
|
||||
"New unknown must be explicitly related to an answer-derived node",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
// Helper that returns a rejectedProposalSnapshot (mirrors the orchestrator logic)
|
||||
function snapshotFromProposal(proposal) {
|
||||
if (!proposal) return null;
|
||||
return {
|
||||
answerMeaning: proposal.answerMeaning
|
||||
? {
|
||||
userSupportedMeaning: proposal.answerMeaning.userSupportedMeaning ?? null,
|
||||
possibleInference: proposal.answerMeaning.possibleInference ?? null,
|
||||
}
|
||||
: null,
|
||||
updatedNodes: (proposal.updatedNodes ?? []).map((n) => ({
|
||||
nodeId: n.nodeId,
|
||||
newValue: n.newValue,
|
||||
})),
|
||||
resolvedUnknownNodeIds: proposal.resolvedUnknownNodeIds ?? [],
|
||||
addedNodes: (proposal.addedNodes ?? []).map((n) => ({
|
||||
id: n.id,
|
||||
kind: n.kind,
|
||||
label: n.label,
|
||||
description: n.description,
|
||||
parentId: n.parentId ?? null,
|
||||
dependsOn: n.dependsOn ?? [],
|
||||
affects: n.affects ?? [],
|
||||
childIds: n.childIds ?? [],
|
||||
})),
|
||||
addedEdges: (proposal.addedEdges ?? []).map((e) => ({
|
||||
fromNodeId: e.fromNodeId,
|
||||
toNodeId: e.toNodeId,
|
||||
relationship: e.relationship,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
// Simulate the orchestrator's failure return path
|
||||
function simulateOrchestratorFailure({ situationGraph, proposal }) {
|
||||
const applicationResult = createCompetitionRejection();
|
||||
const rejectedProposalSnapshot =
|
||||
applicationResult.stage === "proposal_compatibility" && proposal
|
||||
? snapshotFromProposal(proposal)
|
||||
: null;
|
||||
|
||||
return {
|
||||
success: false,
|
||||
stage: applicationResult.stage,
|
||||
errors: applicationResult.errors,
|
||||
diagnostics: { rejectedProposalSnapshot },
|
||||
};
|
||||
}
|
||||
|
||||
// Simulate a non-proposal_compatibility failure (should NOT include snapshot)
|
||||
function simulateNonCompetitionFailure() {
|
||||
return {
|
||||
success: false,
|
||||
stage: "application",
|
||||
errors: ["Could not apply graph update"],
|
||||
diagnostics: {},
|
||||
};
|
||||
}
|
||||
|
||||
describe("rejected proposal compatibility snapshot (orchestrator diagnostics)", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockUpdateCase.mockReset();
|
||||
});
|
||||
|
||||
it("includes rejectedProposalSnapshot when stage is proposal_compatibility", async () => {
|
||||
const situationGraph = { centralStatement: "test", nodes: [], edges: [] };
|
||||
const proposal = {
|
||||
answerMeaning: {
|
||||
userSupportedMeaning: "The user requires evidence for both financial savings and engineering retention.",
|
||||
possibleInference: "Personnel retention is being treated as a veto constraint alongside financial justification.",
|
||||
},
|
||||
updatedNodes: [{ nodeId: "n-x", newValue: "resolved value" }],
|
||||
resolvedUnknownNodeIds: ["n-resolved"],
|
||||
addedNodes: [
|
||||
{
|
||||
id: "n-savings-realism",
|
||||
kind: "unknown",
|
||||
label: "Savings realism",
|
||||
description: "Are projected savings realistic?",
|
||||
parentId: null,
|
||||
dependsOn: [],
|
||||
affects: ["neb1bz2"],
|
||||
childIds: [],
|
||||
},
|
||||
],
|
||||
addedEdges: [
|
||||
{ id: "e-1", fromNodeId: "n-savings-realism", toNodeId: "neb1bz2", relationship: "depends_on", confidence: "medium", description: "dep" },
|
||||
],
|
||||
};
|
||||
|
||||
mockUpdateCase.mockImplementation(async (body) =>
|
||||
simulateOrchestratorFailure(body),
|
||||
);
|
||||
|
||||
const result = await mockUpdateCase({ situationGraph, proposal });
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.stage).toBe("proposal_compatibility");
|
||||
expect(result.diagnostics.rejectedProposalSnapshot).toBeDefined();
|
||||
});
|
||||
|
||||
it("exposes answerMeaning.userSupportedMeaning and possibleInference in the snapshot", async () => {
|
||||
const situationGraph = { centralStatement: "test", nodes: [], edges: [] };
|
||||
const answerMeaningWithMeaning = {
|
||||
userSupportedMeaning: "The user requires evidence for financial savings.",
|
||||
possibleInference: "This is being treated as a hard constraint.",
|
||||
};
|
||||
const proposal = { answerMeaning: answerMeaningWithMeaning, updatedNodes: [], resolvedUnknownNodeIds: [], addedNodes: [], addedEdges: [] };
|
||||
|
||||
mockUpdateCase.mockImplementation(async (body) =>
|
||||
simulateOrchestratorFailure(body),
|
||||
);
|
||||
|
||||
const result = await mockUpdateCase({ situationGraph, proposal });
|
||||
|
||||
expect(result.diagnostics.rejectedProposalSnapshot.answerMeaning.userSupportedMeaning).toBe(
|
||||
"The user requires evidence for financial savings.",
|
||||
);
|
||||
expect(result.diagnostics.rejectedProposalSnapshot.answerMeaning.possibleInference).toBe(
|
||||
"This is being treated as a hard constraint.",
|
||||
);
|
||||
});
|
||||
|
||||
it("exposes added unknown label, description and structural references", async () => {
|
||||
const situationGraph = { centralStatement: "test", nodes: [], edges: [] };
|
||||
const addedNode = {
|
||||
id: "n-savings-realism",
|
||||
kind: "unknown",
|
||||
label: "Are projected savings realistic?",
|
||||
description: "Need evidence that the office savings estimates are defensible.",
|
||||
parentId: null,
|
||||
dependsOn: ["neb1bz2"],
|
||||
affects: [],
|
||||
childIds: [],
|
||||
};
|
||||
const proposal = { answerMeaning: null, updatedNodes: [], resolvedUnknownNodeIds: [], addedNodes: [addedNode], addedEdges: [] };
|
||||
|
||||
mockUpdateCase.mockImplementation(async (body) =>
|
||||
simulateOrchestratorFailure(body),
|
||||
);
|
||||
|
||||
const result = await mockUpdateCase({ situationGraph, proposal });
|
||||
|
||||
expect(result.diagnostics.rejectedProposalSnapshot.addedNodes).toHaveLength(1);
|
||||
const node = result.diagnostics.rejectedProposalSnapshot.addedNodes[0];
|
||||
expect(node.id).toBe("n-savings-realism");
|
||||
expect(node.kind).toBe("unknown");
|
||||
expect(node.label).toBe("Are projected savings realistic?");
|
||||
expect(node.description).toBe("Need evidence that the office savings estimates are defensible.");
|
||||
expect(node.parentId).toBe(null);
|
||||
expect(node.dependsOn).toEqual(["neb1bz2"]);
|
||||
expect(node.affects).toEqual([]);
|
||||
});
|
||||
|
||||
it("exposes added edges with fromNodeId, toNodeId and relationship", async () => {
|
||||
const situationGraph = { centralStatement: "test", nodes: [], edges: [] };
|
||||
const edge = { id: "e-savings-state", fromNodeId: "n-savings-realism", toNodeId: "neb1bz2", relationship: "depends_on", confidence: "medium", description: "dep" };
|
||||
const proposal = { answerMeaning: null, updatedNodes: [], resolvedUnknownNodeIds: [], addedNodes: [], addedEdges: [edge] };
|
||||
|
||||
mockUpdateCase.mockImplementation(async (body) =>
|
||||
simulateOrchestratorFailure(body),
|
||||
);
|
||||
|
||||
const result = await mockUpdateCase({ situationGraph, proposal });
|
||||
|
||||
expect(result.diagnostics.rejectedProposalSnapshot.addedEdges).toHaveLength(1);
|
||||
const e = result.diagnostics.rejectedProposalSnapshot.addedEdges[0];
|
||||
expect(e.fromNodeId).toBe("n-savings-realism");
|
||||
expect(e.toNodeId).toBe("neb1bz2");
|
||||
expect(e.relationship).toBe("depends_on");
|
||||
});
|
||||
|
||||
it("retains existing rejection stage and errors unchanged", async () => {
|
||||
const situationGraph = { centralStatement: "test", nodes: [], edges: [] };
|
||||
const proposal = { answerMeaning: null, updatedNodes: [], resolvedUnknownNodeIds: [], addedNodes: [], addedEdges: [] };
|
||||
|
||||
mockUpdateCase.mockImplementation(async (body) =>
|
||||
simulateOrchestratorFailure(body),
|
||||
);
|
||||
|
||||
const result = await mockUpdateCase({ situationGraph, proposal });
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.stage).toBe("proposal_compatibility");
|
||||
expect(result.errors).toEqual([
|
||||
"answerMeaning.userSupportedMeaning introduces a stronger reasoning category than the raw answer establishes.",
|
||||
"New unknown must be explicitly related to an answer-derived node",
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not include raw model response or prompt in the snapshot", async () => {
|
||||
const situationGraph = { centralStatement: "test", nodes: [], edges: [] };
|
||||
const proposal = { answerMeaning: null, updatedNodes: [], resolvedUnknownNodeIds: [], addedNodes: [], addedEdges: [] };
|
||||
|
||||
mockUpdateCase.mockImplementation(async (body) =>
|
||||
simulateOrchestratorFailure(body),
|
||||
);
|
||||
|
||||
const result = await mockUpdateCase({ situationGraph, proposal });
|
||||
|
||||
const snapshotKeys = Object.keys(result.diagnostics.rejectedProposalSnapshot);
|
||||
for (const key of snapshotKeys) {
|
||||
expect(key).not.toContain("raw");
|
||||
expect(key).not.toContain("prompt");
|
||||
expect(key).not.toContain("chain_of_thought");
|
||||
expect(key).not.toContain("provider_metadata");
|
||||
}
|
||||
});
|
||||
|
||||
it("does not include rejectedProposalSnapshot for non-proposal_compatibility failures", async () => {
|
||||
mockUpdateCase.mockImplementation(async () =>
|
||||
simulateNonCompetitionFailure(),
|
||||
);
|
||||
|
||||
const result = await mockUpdateCase({ situationGraph: {}, proposal: {} });
|
||||
|
||||
expect(result.diagnostics.rejectedProposalSnapshot).toBeUndefined();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user