robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: surface new unknowns after graph updates

Browse Source
This commit is contained in:
robbond
2026-08-02 10:30:59 +01:00
parent 904aec7616
commit 72ef175971
16 changed files with 910 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/app/api/cases-update-route.test.js
+3
View File
@@ -25,7 +25,9 @@ function makeSuccessResult() {
removedEdgeIds: [],
resolvedUnknownNodeIds: ["n1"],
affectedNodeIds: ["n1"],
selectedQuestion: null,
},
selectedQuestion: null,
affectedNodeIds: ["n1"],
resolvedUnknownNodeIds: ["n1"],
previousActiveUnknownNodeId: "n0",
@@ -268,6 +270,7 @@ describe("app/api/cases/update route", () => {
resolvedUnknownNodeIds: success.resolvedUnknownNodeIds,
previousActiveUnknownNodeId: success.previousActiveUnknownNodeId,
newActiveUnknownNodeId: success.newActiveUnknownNodeId,
selectedQuestion: success.selectedQuestion,
changesApplied: success.changesApplied,
diagnostics: success.diagnostics,
});
tests/graph/apply-proposal.test.js
+244
View File
@@ -99,6 +99,7 @@ function makeApplicationFixture() {
removedEdgeIds: [],
resolvedUnknownNodeIds: [complaintRateUnknown.id],
affectedNodeIds: [qualityDeterioration.id],
selectedQuestion: null,
};
return {
@@ -444,6 +445,7 @@ describe("applyValidatedProposal", () => {
removedEdgeIds: [],
resolvedUnknownNodeIds: [],
affectedNodeIds: [],
selectedQuestion: null,
},
});
@@ -455,4 +457,246 @@ describe("applyValidatedProposal", () => {
expect.arrayContaining([expect.stringContaining("no meaningful change")]),
);
});
it("resolves one unknown and adds consequential unknowns with one selected question", () => {
const { graph, ids } = makeApplicationFixture();
const proposal = {
addedNodes: [
makeNode({
id: "n-commercial-value",
label: "Commercial value definition",
description:
"Need a concrete definition of commercial value because the decision depends on it.",
kind: "unknown",
status: "unknown",
confidence: "high",
}),
makeNode({
id: "n-demand-evidence",
label: "Evidence of demand",
description:
"Need evidence of demand because it matters to the build decision.",
kind: "unknown",
status: "unknown",
confidence: "medium",
}),
makeNode({
id: "n-build-decision",
label: "Build Confidence Engine decision",
description: "Decision situation introduced by the answer.",
kind: "state",
status: "supported",
confidence: "medium",
}),
],
updatedNodes: [
{
nodeId: ids.complaintRateUnknown,
previousStatus: "unknown",
newStatus: "resolved",
previousValue: null,
newValue: "Decision whether to build Confidence Engine",
reason: "The answer resolves the original context unknown.",
},
],
addedEdges: [
makeEdge({
id: "e-build-commercial-value",
fromNodeId: "n-build-decision",
toNodeId: "n-commercial-value",
relationship: "depends_on",
confidence: "medium",
description: "The decision depends on defining commercial value.",
}),
makeEdge({
id: "e-build-demand-evidence",
fromNodeId: "n-build-decision",
toNodeId: "n-demand-evidence",
relationship: "depends_on",
confidence: "medium",
description: "The decision depends on evidence of demand.",
}),
],
removedEdgeIds: [],
resolvedUnknownNodeIds: [ids.complaintRateUnknown],
affectedNodeIds: [],
selectedQuestion: {
nodeId: "n-commercial-value",
question: "How should commercial value be defined for this decision?",
reason:
"This is the most consequential unresolved unknown introduced by the answer.",
},
};
const result = applyValidatedProposal({ situationGraph: graph, proposal });
expect(result.success).toBe(true);
expect(result.updatedSituationGraph.resolvedNodeIds).toContain(
ids.complaintRateUnknown,
);
expect(
result.updatedSituationGraph.nodes.some(
(node) => node.id === "n-commercial-value",
),
).toBe(true);
expect(
result.updatedSituationGraph.nodes.some(
(node) => node.id === "n-demand-evidence",
),
).toBe(true);
expect(result.newActiveUnknownNodeId).toBe("n-commercial-value");
expect(result.selectedQuestion).toEqual(proposal.selectedQuestion);
});
it("rejects more than 3 added unknowns", () => {
const { graph, proposal, ids } = makeApplicationFixture();
const result = applyValidatedProposal({
situationGraph: graph,
proposal: {
...proposal,
addedNodes: [1, 2, 3, 4].map((index) =>
makeNode({
id: `n-unknown-${index}`,
label: `Unknown ${index}`,
description: `Need unknown ${index} because it matters to the decision.`,
kind: "unknown",
status: "unknown",
confidence: "medium",
}),
),
addedEdges: [1, 2, 3, 4].map((index) =>
makeEdge({
id: `e-unknown-${index}`,
fromNodeId: ids.complaintRateUnknown,
toNodeId: `n-unknown-${index}`,
relationship: "depends_on",
confidence: "medium",
description: `Links unknown ${index}`,
}),
),
selectedQuestion: {
nodeId: "n-unknown-1",
question: "What is unknown 1?",
reason: "Follow-up required.",
},
},
});
expect(result.success).toBe(false);
expect(result.errors.join(" ")).toContain("too many unknown nodes");
});
it("rejects unrelated added unknowns", () => {
const { graph, proposal } = makeApplicationFixture();
const result = applyValidatedProposal({
situationGraph: graph,
proposal: {
...proposal,
addedNodes: [
makeNode({
id: "n-unrelated",
label: "Office rent",
description:
"Need office rent because it matters to a different branch.",
kind: "unknown",
status: "unknown",
confidence: "low",
}),
],
selectedQuestion: {
nodeId: "n-unrelated",
question: "What is the office rent?",
reason: "Unrelated test.",
},
},
});
expect(result.success).toBe(false);
expect(result.errors.join(" ")).toContain(
"explicitly related to an answer-derived node",
);
});
it("rejects selected question referencing resolved node", () => {
const { graph, proposal, ids } = makeApplicationFixture();
const result = applyValidatedProposal({
situationGraph: graph,
proposal: {
...proposal,
selectedQuestion: {
nodeId: ids.complaintRateUnknown,
question: "What is the complaint rate?",
reason: "Invalid reselection.",
},
},
});
expect(result.success).toBe(false);
expect(result.errors.join(" ")).toContain(
"selectedQuestion must reference an unresolved node",
);
});
it("active unknown matches selected question node", () => {
const { graph, ids } = makeApplicationFixture();
const proposal = {
addedNodes: [
makeNode({
id: "n-success-threshold",
label: "Success threshold",
description:
"Need a success threshold because the decision depends on it.",
kind: "unknown",
status: "unknown",
confidence: "high",
}),
makeNode({
id: "n-build-decision",
label: "Build Confidence Engine decision",
description: "Decision introduced by the answer.",
kind: "state",
status: "supported",
confidence: "medium",
}),
],
updatedNodes: [
{
nodeId: ids.complaintRateUnknown,
previousStatus: "unknown",
newStatus: "resolved",
previousValue: null,
newValue: "Decision whether to build Confidence Engine",
reason: "The answer resolves the original unknown.",
},
],
addedEdges: [
makeEdge({
id: "e-build-success-threshold",
fromNodeId: "n-build-decision",
toNodeId: "n-success-threshold",
relationship: "depends_on",
confidence: "medium",
description: "The decision depends on a success threshold.",
}),
],
removedEdgeIds: [],
resolvedUnknownNodeIds: [ids.complaintRateUnknown],
affectedNodeIds: [],
selectedQuestion: {
nodeId: "n-success-threshold",
question: "What success threshold would justify building it?",
reason: "One consequential unknown remains.",
},
};
const result = applyValidatedProposal({ situationGraph: graph, proposal });
expect(result.success).toBe(true);
expect(result.newActiveUnknownNodeId).toBe(result.selectedQuestion?.nodeId);
});
});
tests/graph/orchestrator.test.js
+61
View File
@@ -113,6 +113,7 @@ function makeProposal(overrides = {}) {
removedEdgeIds: [],
resolvedUnknownNodeIds: ["n-unknown"],
affectedNodeIds: [],
selectedQuestion: null,
...overrides,
};
}
@@ -529,6 +530,66 @@ describe("lib/graph/orchestrator startCase", () => {
expect(result.proposal.nextQuestion).toBeUndefined();
});
it("returns selectedQuestion from applied update proposal", async () => {
const { updateCase } = await import("@/lib/graph/orchestrator.js");
const provider = {
generateReconstruction: vi.fn().mockResolvedValue(
makeProposal({
addedNodes: [
makeNode({
id: "n-build-decision",
label: "Build Confidence Engine decision",
description: "Decision introduced by the answer.",
kind: "state",
status: "supported",
confidence: "medium",
}),
makeNode({
id: "n-commercial-value",
label: "Commercial value definition",
description:
"Need a concrete definition because the decision depends on it.",
kind: "unknown",
status: "unknown",
confidence: "high",
}),
],
addedEdges: [
{
id: "e-build-commercial-value",
fromNodeId: "n-build-decision",
toNodeId: "n-commercial-value",
relationship: "depends_on",
confidence: "medium",
description:
"The decision depends on commercial value definition.",
},
],
selectedQuestion: {
nodeId: "n-commercial-value",
question:
"How should commercial value be defined for this decision?",
reason: "Consequential unresolved uncertainty remains.",
},
}),
),
};
const result = await updateCase(makeUpdateRequest(), {
provider,
config: MOCK_CONFIG,
applyProposal: true,
});
expect(result.success).toBe(true);
expect(result.selectedQuestion).toEqual({
nodeId: "n-commercial-value",
question: "How should commercial value be defined for this decision?",
reason: "Consequential unresolved uncertainty remains.",
});
expect(result.newActiveUnknownNodeId).toBe("n-commercial-value");
});
it("defaults to proposal-only mode", async () => {
const { updateCase } = await import("@/lib/graph/orchestrator.js");
const applyValidatedProposal = vi.fn();
tests/graph/prompt-builder.test.js
+10
View File
@@ -72,6 +72,7 @@ describe("buildGraphUpdatePrompt", () => {
expect(prompt).toContain("removedEdgeIds");
expect(prompt).toContain("resolvedUnknownNodeIds");
expect(prompt).toContain("affectedNodeIds");
expect(prompt).toContain("selectedQuestion");
});
it("lists enum values", () => {
@@ -98,4 +99,13 @@ describe("buildGraphUpdatePrompt", () => {
expect(prompt).toContain("Return JSON only");
expect(prompt).toContain("Return one JSON object only");
});
it("describes controlled emergent unknown rules", () => {
const prompt = buildGraphUpdatePrompt(makeContext());
expect(prompt).toContain("Add at most 3 new unknown nodes");
expect(prompt).toContain("Resolve the answered unknown first");
expect(prompt).toContain(
"selectedQuestion.question must be one narrow non-compound question",
);
});
});
tests/graph/schema.test.js
+85 -15
View File
@@ -161,21 +161,56 @@ describe("graphUpdateSchema", () => {
it("validates a complete update", () => {
const node = makeNode({ id: "n2", label: "New Node" });
const edge = makeEdge({ fromNodeId: "n1", toNodeId: "n2" });
const result = graphUpdateSchema.safeParse({
addedNodes: [node],
updatedNodes: [{ nodeId: "n1", newStatus: "resolved", previousStatus: "unknown", reason: "Question answered" }],
updatedNodes: [
{
nodeId: "n1",
newStatus: "resolved",
previousStatus: "unknown",
reason: "Question answered",
},
],
addedEdges: [edge],
removedEdgeIds: ["e-old"],
resolvedUnknownNodeIds: ["n2"],
affectedNodeIds: ["n3"],
selectedQuestion: {
nodeId: "n2",
question: "What does this new node mean?",
reason: "A follow-up unknown remains.",
},
});
expect(result.success).toBe(true);
});
it("allows null selectedQuestion", () => {
const result = graphUpdateSchema.safeParse({
selectedQuestion: null,
});
expect(result.success).toBe(true);
});
it("rejects update with invalid node kind in addedNodes", () => {
const invalid = graphUpdateSchema.safeParse({
addedNodes: [{ id: "x", label: "Test", kind: "invalid_kind", description: "test", status: "unknown", confidence: "medium", value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], parentId: null, childIds: [] }],
addedNodes: [
{
id: "x",
label: "Test",
kind: "invalid_kind",
description: "test",
status: "unknown",
confidence: "medium",
value: null,
unit: null,
evidenceIds: [],
dependsOn: [],
affects: [],
parentId: null,
childIds: [],
},
],
});
expect(invalid.success).toBe(false);
});
@@ -184,7 +219,9 @@ describe("graphUpdateSchema", () => {
describe("API request schemas", () => {
describe("startCaseRequestSchema", () => {
it("validates scenario field", () => {
const result = startCaseRequestSchema.safeParse({ scenario: "Test scenario" });
const result = startCaseRequestSchema.safeParse({
scenario: "Test scenario",
});
expect(result.success).toBe(true);
});
@@ -195,14 +232,16 @@ describe("API request schemas", () => {
it("rejects scenario over 10000 chars", () => {
const longScenario = "a".repeat(10001);
const result = startCaseRequestSchema.safeParse({ scenario: longScenario });
const result = startCaseRequestSchema.safeParse({
scenario: longScenario,
});
expect(result.success).toBe(false);
});
it("accepts optional promptVersion", () => {
const result = startCaseRequestSchema.safeParse({
scenario: "Test",
promptVersion: "v0.3"
const result = startCaseRequestSchema.safeParse({
scenario: "Test",
promptVersion: "v0.3",
});
expect(result.success).toBe(true);
});
@@ -213,7 +252,7 @@ describe("API request schemas", () => {
const graph = makeGraph({
centralStatement: "Test scenario",
nodes: [makeNode({ id: "n1", label: "N" })],
currentSummary: "Current state of situation"
currentSummary: "Current state of situation",
});
const result = updateCaseRequestSchema.safeParse({
situationGraph: graph,
@@ -235,7 +274,7 @@ describe("API request schemas", () => {
const graph = makeGraph({
centralStatement: "Test",
nodes: [makeNode({ id: "n1", label: "N" })],
currentSummary: "Test summary"
currentSummary: "Test summary",
});
const result = updateCaseRequestSchema.safeParse({
situationGraph: graph,
@@ -261,7 +300,9 @@ describe("deterministic ID generation", () => {
});
it("IDs are prefixed with 'n' and short", () => {
const id = makeNodeId("A very long label that would produce a longer hash if not truncated");
const id = makeNodeId(
"A very long label that would produce a longer hash if not truncated",
);
expect(id.startsWith("n")).toBe(true);
expect(id.length).toBeLessThan(15);
});
@@ -325,7 +366,7 @@ describe("helper functions", () => {
const graph = makeGraph({
centralStatement: "Test",
currentSummary: "Default summary",
nodes: [makeNode({ id: "n1", label: "Placeholder" })]
nodes: [makeNode({ id: "n1", label: "Placeholder" })],
});
const result = situationGraphSchema.safeParse(graph);
expect(result.success).toBe(true);
@@ -346,19 +387,48 @@ describe("helper functions", () => {
describe("enum values completeness", () => {
it("SituationKind has all expected values", () => {
const expected = ["observation", "reported_claim", "metric", "state", "transition", "relationship", "assumption", "unknown", "conclusion"];
const expected = [
"observation",
"reported_claim",
"metric",
"state",
"transition",
"relationship",
"assumption",
"unknown",
"conclusion",
];
const actual = Object.values(SituationKind);
expect(actual).toEqual(expect.arrayContaining(expected));
});
it("SituationStatus has all expected values", () => {
const expected = ["known", "unknown", "provisional", "supported", "weakened", "contradicted", "resolved"];
const expected = [
"known",
"unknown",
"provisional",
"supported",
"weakened",
"contradicted",
"resolved",
];
const actual = Object.values(SituationStatus);
expect(actual).toEqual(expect.arrayContaining(expected));
});
it("SituationRelationship has all expected values", () => {
const expected = ["supports", "weakens", "contradicts", "depends_on", "causes", "may_cause", "measures", "compares_with", "updates", "other"];
const expected = [
"supports",
"weakens",
"contradicts",
"depends_on",
"causes",
"may_cause",
"measures",
"compares_with",
"updates",
"other",
];
const actual = Object.values(SituationRelationship);
expect(actual).toEqual(expect.arrayContaining(expected));
});
tests/graph/update-proposal.test.js
+29 -1
View File
@@ -18,6 +18,7 @@ function makeValidProposal(overrides = {}) {
removedEdgeIds: [],
resolvedUnknownNodeIds: ["n-unknown"],
affectedNodeIds: [],
selectedQuestion: null,
...overrides,
};
}
@@ -117,7 +118,34 @@ describe("parseGraphUpdateProposal", () => {
expect(result.success).toBe(false);
});
it("does not invent a next question", () => {
it("defaults missing selectedQuestion to null", () => {
const result = parseGraphUpdateProposal({
addedNodes: [],
updatedNodes: [],
addedEdges: [],
removedEdgeIds: [],
resolvedUnknownNodeIds: [],
affectedNodeIds: [],
});
expect(result.success).toBe(true);
expect(result.proposal.selectedQuestion).toBeNull();
});
it("parses a valid selectedQuestion", () => {
const result = parseGraphUpdateProposal(
makeValidProposal({
selectedQuestion: {
nodeId: "n-follow-up",
question: "How should commercial value be defined for this decision?",
reason: "A consequential unknown remains unresolved.",
},
}),
);
expect(result.success).toBe(true);
expect(result.proposal.selectedQuestion?.nodeId).toBe("n-follow-up");
});
it("does not invent a next question field outside the contract", () => {
const result = parseGraphUpdateProposal(makeValidProposal());
expect(result.proposal.nextQuestion).toBeUndefined();
});
tests/smoke.test.js
+7 -1
View File
@@ -64,9 +64,15 @@ test("graph-backed one-turn update smoke test", async ({ page }) => {
timeout: 240000,
});
await expect(page.getByText(/Resolved unknowns/i)).toBeVisible();
await expect(page.getByText(/Newly surfaced unknowns/i)).toBeVisible();
await expect(page.getByText(/Affected nodes/i)).toBeVisible();
await expect(
page.getByText(/No next question selected yet\./i),
page.getByText(/Selected Question|Next question:/i),
).toBeVisible();
await expect(
page.getByText(
/Additional submission is disabled in this one-update prototype\./i,
),
).toBeVisible();
await expect(page.getByText(/Error:/i)).toHaveCount(0);
await expect(page.getByText(/Update error:/i)).toHaveCount(0);
tests/ui/scenario-form.test.jsx
+100 -4
View File
@@ -113,11 +113,37 @@ function makeUpdateSuccess(overrides = {}) {
value: "1.9 complaints per 100 units",
unit: null,
},
{
id: "n-next-unknown",
label: "Commercial value definition",
description: "Need a definition because the decision depends on it.",
kind: "unknown",
status: "unknown",
confidence: "high",
value: null,
unit: null,
},
],
edges: [],
},
proposal: {
addedNodes: [],
addedNodes: [
{
id: "n-next-unknown",
label: "Commercial value definition",
description: "Need a definition because the decision depends on it.",
kind: "unknown",
status: "unknown",
confidence: "high",
value: null,
unit: null,
evidenceIds: [],
dependsOn: [],
affects: [],
parentId: null,
childIds: [],
},
],
updatedNodes: [
{ nodeId: "n-unknown", newStatus: "resolved", reason: "answered" },
],
@@ -125,6 +151,16 @@ function makeUpdateSuccess(overrides = {}) {
removedEdgeIds: [],
resolvedUnknownNodeIds: ["n-unknown"],
affectedNodeIds: ["n-conclusion"],
selectedQuestion: {
nodeId: "n-next-unknown",
question: "How should commercial value be defined for this decision?",
reason: "A narrower consequential uncertainty remains.",
},
},
selectedQuestion: {
nodeId: "n-next-unknown",
question: "How should commercial value be defined for this decision?",
reason: "A narrower consequential uncertainty remains.",
},
affectedNodeIds: ["n-conclusion"],
resolvedUnknownNodeIds: ["n-unknown"],
@@ -323,6 +359,20 @@ describe("graph-backed UI rendering", () => {
expect(html).toContain("Complaint rate denominator");
});
it("newly surfaced unknowns render", () => {
const html = renderToStaticMarkup(
<GraphUpdateView
updateResult={{
...makeUpdateSuccess(),
previousSituationGraph: makeGraphResult().situationGraph,
}}
/>,
);
expect(html).toContain("Newly surfaced unknowns");
expect(html).toContain("Commercial value definition");
});
it("affected nodes render", () => {
const html = renderToStaticMarkup(
<GraphUpdateView
@@ -337,11 +387,33 @@ describe("graph-backed UI rendering", () => {
expect(html).toContain("Quality deterioration");
});
it("no fake next question appears", () => {
it("renders validated next question when present", () => {
const html = renderToStaticMarkup(
<GraphUpdateView
updateResult={{
...makeUpdateSuccess({ newActiveUnknownNodeId: null }),
...makeUpdateSuccess(),
previousSituationGraph: makeGraphResult().situationGraph,
}}
/>,
);
expect(html).toContain(
"How should commercial value be defined for this decision?",
);
});
it("no fake next question appears when there is none", () => {
const html = renderToStaticMarkup(
<GraphUpdateView
updateResult={{
...makeUpdateSuccess({
newActiveUnknownNodeId: null,
selectedQuestion: null,
proposal: {
...makeUpdateSuccess().proposal,
selectedQuestion: null,
},
}),
previousSituationGraph: makeGraphResult().situationGraph,
}}
/>,
@@ -363,7 +435,31 @@ describe("graph-backed UI rendering", () => {
expect(html).toContain("Previous active unknown");
expect(html).toContain("Complaint rate denominator");
expect(html).toContain("New active unknown");
expect(html).toContain("Unknown node (ID: n-next-unknown)");
expect(html).toContain("Commercial value definition");
});
it("situation graph marks newly surfaced and active unknowns", () => {
const html = renderToStaticMarkup(
<SituationGraphView
situationGraph={makeUpdateSuccess().updatedSituationGraph}
selectedQuestion={makeUpdateSuccess().selectedQuestion}
newlySurfacedNodeIds={["n-next-unknown"]}
/>,
);
expect(html).toContain("newly surfaced unknown");
expect(html).toContain("active unknown");
expect(html).toContain("resolved unknown");
});
it("disabled follow-up form is shown only as prototype limitation", () => {
const html = renderToStaticMarkup(
<GraphUpdateView
updateResult={makeUpdateSuccess()}
/>,
);
expect(html).toContain("How should commercial value be defined for this decision?");
});
it("raw ids remain only in collapsed proposal details", () => {