robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: explain deterministic unknown selection

Browse Source
This commit is contained in:
robbond
2026-08-02 15:27:00 +01:00
parent 5ef9710293
commit 586802950d
3 changed files with 321 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/graph/orchestrator.js
+28 -1
View File
@@ -17,6 +17,7 @@ import { applyValidatedProposal } from "./apply-proposal.js";
import { buildGraphUpdatePrompt } from "./prompt-builder.js";
import { parseGraphUpdateProposal } from "./update-proposal.js";
import {
explainUnknownSelection,
selectActiveUnknownCandidate,
validateGraphReferences,
} from "./utils.js";
@@ -31,7 +32,12 @@ function toValidationErrors(error) {
);
}
function buildDiagnostics({ analysis, graph, graphReferenceValidation }) {
function buildDiagnostics({
analysis,
graph,
graphReferenceValidation,
unknownSelectionExplanation,
}) {
return {
promptVersion: analysis?.promptVersion ?? null,
modelName: analysis?.modelName ?? null,
@@ -43,6 +49,7 @@ function buildDiagnostics({ analysis, graph, graphReferenceValidation }) {
compatibilityApplied: analysis?.compatibilityApplied ?? false,
compatibilityChanges: analysis?.compatibilityChanges ?? [],
compatibilityWarnings: analysis?.compatibilityWarnings ?? [],
unknownSelectionExplanation: unknownSelectionExplanation ?? null,
};
}
@@ -54,6 +61,7 @@ function buildUpdateDiagnostics({
graph,
graphReferenceValidation,
selectedQuestion,
unknownSelectionExplanation,
}) {
return {
promptVersion: promptVersion ?? "v0.4",
@@ -71,6 +79,7 @@ function buildUpdateDiagnostics({
selectedQuestion?.investigationStrategy ??
selectedQuestion?.strategy ??
null,
unknownSelectionExplanation: unknownSelectionExplanation ?? null,
};
}
@@ -131,6 +140,10 @@ export async function startCase(body) {
situationGraphSchema.parse(situationGraph);
const graphReferenceValidation = validateGraphReferences(situationGraph);
const unknownSelectionExplanation = explainUnknownSelection(
situationGraph,
[],
);
if (!graphReferenceValidation.valid) {
return {
success: false,
@@ -139,6 +152,7 @@ export async function startCase(body) {
analysis,
graph: situationGraph,
graphReferenceValidation,
unknownSelectionExplanation,
}),
validationErrors: graphReferenceValidation.errors,
statusCode: 500,
@@ -153,6 +167,7 @@ export async function startCase(body) {
analysis,
graph: situationGraph,
graphReferenceValidation,
unknownSelectionExplanation,
}),
};
}
@@ -290,6 +305,10 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
graph: situationGraph,
graphReferenceValidation: graphReferenceValidation,
selectedQuestion: null,
unknownSelectionExplanation: explainUnknownSelection(
situationGraph,
situationGraph.resolvedNodeIds || [],
),
}),
},
statusCode:
@@ -320,6 +339,10 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
graph: applicationResult.updatedSituationGraph,
graphReferenceValidation: applicationResult.graphReferenceValidation,
selectedQuestion: applicationResult.selectedQuestion,
unknownSelectionExplanation: explainUnknownSelection(
applicationResult.updatedSituationGraph,
applicationResult.updatedSituationGraph.resolvedNodeIds || [],
),
}),
};
}
@@ -336,6 +359,10 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
graph: situationGraph,
graphReferenceValidation,
selectedQuestion: null,
unknownSelectionExplanation: explainUnknownSelection(
situationGraph,
situationGraph.resolvedNodeIds || [],
),
}),
};
}
lib/graph/utils.js
+227 -24
View File
@@ -95,6 +95,127 @@ function classifyUnknownPriority(text) {
return matches;
}
function buildScoreContributions(
matches,
downstreamCount,
unresolvedParentUnknownCount,
) {
const contributions = [
{
rule: "downstream_dependencies",
value: downstreamCount,
weight: 4,
delta: downstreamCount * 4,
},
];
if (matches.objective) {
contributions.push({
rule: "objective_match",
value: true,
weight: 12,
delta: 12,
});
}
if (matches.actor) {
contributions.push({
rule: "actor_match",
value: true,
weight: 10,
delta: 10,
});
}
if (matches.criteria) {
contributions.push({
rule: "criteria_match",
value: true,
weight: 11,
delta: 11,
});
}
if (matches.measure) {
contributions.push({
rule: "measure_match",
value: true,
weight: 8,
delta: 8,
});
}
if (matches.terminology) {
contributions.push({
rule: "terminology_match",
value: true,
weight: 7,
delta: 7,
});
}
if (matches.constraint) {
contributions.push({
rule: "constraint_match",
value: true,
weight: 9,
delta: 9,
});
}
if (matches.pricing) {
contributions.push({
rule: "pricing_penalty",
value: true,
weight: -8,
delta: -8,
});
}
if (matches.implementation) {
contributions.push({
rule: "implementation_penalty",
value: true,
weight: -10,
delta: -10,
});
}
if (matches.optimisation) {
contributions.push({
rule: "optimisation_penalty",
value: true,
weight: -9,
delta: -9,
});
}
if (matches.speculative) {
contributions.push({
rule: "speculative_penalty",
value: true,
weight: -12,
delta: -12,
});
}
if (
matches.pricing &&
!matches.objective &&
!matches.criteria &&
!matches.actor
) {
contributions.push({
rule: "isolated_pricing_penalty",
value: true,
weight: -6,
delta: -6,
});
}
if (unresolvedParentUnknownCount > 0) {
contributions.push({
rule: "unresolved_prerequisite_penalty",
value: unresolvedParentUnknownCount,
weight: -7,
delta: unresolvedParentUnknownCount * -7,
});
}
return contributions;
}
export function scoreUnknownCandidate(graph, node, resolvedNodeIds = []) {
const text = collectNodeText(node);
const matches = classifyUnknownPriority(text);
@@ -105,30 +226,15 @@ export function scoreUnknownCandidate(graph, node, resolvedNodeIds = []) {
resolvedNodeIds,
);
let score = downstreamCount * 4;
if (matches.objective) score += 12;
if (matches.actor) score += 10;
if (matches.criteria) score += 11;
if (matches.measure) score += 8;
if (matches.terminology) score += 7;
if (matches.constraint) score += 9;
if (matches.pricing) score -= 8;
if (matches.implementation) score -= 10;
if (matches.optimisation) score -= 9;
if (matches.speculative) score -= 12;
if (
matches.pricing &&
!matches.objective &&
!matches.criteria &&
!matches.actor
) {
score -= 6;
}
score -= unresolvedParentUnknownCount * 7;
const contributions = buildScoreContributions(
matches,
downstreamCount,
unresolvedParentUnknownCount,
);
const score = contributions.reduce(
(total, contribution) => total + contribution.delta,
0,
);
return {
nodeId: node.id,
@@ -137,6 +243,7 @@ export function scoreUnknownCandidate(graph, node, resolvedNodeIds = []) {
downstreamCount,
unresolvedParentUnknownCount,
matches,
contributions,
};
}
@@ -406,6 +513,102 @@ export function selectActiveUnknownCandidate(graph, resolvedNodeIds) {
};
}
export function explainUnknownSelection(graph, resolvedNodeIds = []) {
const unresolved = graph.nodes.filter(
(n) => n.kind === "unknown" && !resolvedNodeIds.includes(n.id),
);
if (unresolved.length === 0) {
return {
selectedNodeId: null,
selectedNodeLabel: null,
resolvedNodeIds: [...resolvedNodeIds],
candidates: [],
competitors: [],
tieBreakOrder: [
"score_desc",
"downstreamCount_desc",
"unresolvedParentUnknownCount_asc",
"label_asc",
],
summary: {
candidateCount: 0,
},
};
}
const candidates = unresolved.map((node) => ({
nodeId: node.id,
label: node.label,
...scoreUnknownCandidate(graph, node, resolvedNodeIds),
}));
candidates.sort((a, b) => {
if (b.score !== a.score) return b.score - a.score;
if (b.downstreamCount !== a.downstreamCount) {
return b.downstreamCount - a.downstreamCount;
}
if (a.unresolvedParentUnknownCount !== b.unresolvedParentUnknownCount) {
return a.unresolvedParentUnknownCount - b.unresolvedParentUnknownCount;
}
return a.label.localeCompare(b.label);
});
const selected = candidates[0];
const competitors = candidates.slice(1).map((candidate) => ({
nodeId: candidate.nodeId,
label: candidate.label,
score: candidate.score,
downstreamCount: candidate.downstreamCount,
unresolvedParentUnknownCount: candidate.unresolvedParentUnknownCount,
matches: candidate.matches,
contributions: candidate.contributions,
outrankedBy: {
scoreDelta: selected.score - candidate.score,
downstreamDelta: selected.downstreamCount - candidate.downstreamCount,
unresolvedPrerequisiteDelta:
candidate.unresolvedParentUnknownCount -
selected.unresolvedParentUnknownCount,
labelOrderWinner:
selected.score === candidate.score &&
selected.downstreamCount === candidate.downstreamCount &&
selected.unresolvedParentUnknownCount ===
candidate.unresolvedParentUnknownCount
? selected.label.localeCompare(candidate.label) <= 0
? selected.label
: candidate.label
: null,
},
}));
return {
selectedNodeId: selected.nodeId,
selectedNodeLabel: selected.label,
resolvedNodeIds: [...resolvedNodeIds],
tieBreakOrder: [
"score_desc",
"downstreamCount_desc",
"unresolvedParentUnknownCount_asc",
"label_asc",
],
candidates,
selected: {
nodeId: selected.nodeId,
label: selected.label,
score: selected.score,
downstreamCount: selected.downstreamCount,
unresolvedParentUnknownCount: selected.unresolvedParentUnknownCount,
matches: selected.matches,
contributions: selected.contributions,
},
competitors,
summary: {
candidateCount: candidates.length,
selectedReason: `highest_score=${selected.score}; downstream=${selected.downstreamCount}; unresolved_prerequisites=${selected.unresolvedParentUnknownCount}`,
},
};
}
// ── Apply a graph update deterministically ──
export function applyGraphUpdate(graph, update) {
tests/graph/orchestrator.test.js
+66
View File
@@ -178,6 +178,9 @@ describe("lib/graph/orchestrator startCase", () => {
expect(result.success).toBe(true);
expect(result.situationGraph.activeUnknownNodeId).toBeTruthy();
expect(result.diagnostics.unknownSelectionExplanation?.selectedNodeId).toBe(
result.situationGraph.activeUnknownNodeId,
);
});
it("returns structured failure when graph reference validation fails", async () => {
@@ -262,6 +265,69 @@ describe("lib/graph/orchestrator startCase", () => {
expect(result.diagnostics.compatibilityChanges).toHaveLength(1);
});
it("preserves selected question bytes while adding selection explanation diagnostics", async () => {
const { updateCase } = await import("@/lib/graph/orchestrator.js");
const provider = {
generateReconstruction: vi.fn().mockResolvedValue(
makeProposal({
addedNodes: [
makeNode({
id: "n-build-decision",
label: "Build Confidence Engine decision",
description: "Decision introduced by the answer.",
kind: "state",
status: "supported",
confidence: "medium",
}),
makeNode({
id: "n-commercial-value",
label: "Commercial value definition",
description:
"Need a concrete definition because the decision depends on it.",
kind: "unknown",
status: "unknown",
confidence: "high",
}),
],
addedEdges: [
{
id: "e-build-commercial-value",
fromNodeId: "n-build-decision",
toNodeId: "n-commercial-value",
relationship: "depends_on",
confidence: "medium",
description:
"The decision depends on commercial value definition.",
},
],
selectedQuestion: {
nodeId: "n-commercial-value",
question:
"How should commercial value be defined for this decision?",
reason: "Consequential unresolved uncertainty remains.",
},
}),
),
};
const first = await updateCase(makeUpdateRequest(), {
provider,
config: MOCK_CONFIG,
applyProposal: true,
});
const second = await updateCase(makeUpdateRequest(), {
provider,
config: MOCK_CONFIG,
applyProposal: true,
});
expect(first.selectedQuestion.question).toBe(
second.selectedQuestion.question,
);
expect(first.selectedQuestion.reason).toBe(second.selectedQuestion.reason);
expect(first.diagnostics.unknownSelectionExplanation).toBeTruthy();
});
it("produces a validated update proposal for a valid request", async () => {
const { updateCase } = await import("@/lib/graph/orchestrator.js");
const provider = {