robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: handle unjustified unknown selection ties

Browse Source
This commit is contained in:
robbond
2026-08-02 16:09:00 +01:00
parent a1f6d0c2b9
commit 51ce356218
8 changed files with 731 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/graph/apply-proposal.js
+35 -17
View File
@@ -1,5 +1,8 @@
import { describeGraph } from "./builder.js";
import { formulateQuestion } from "./question-formulator.js";
import {
formulateQuestion,
formulateTieResolutionQuestion,
} from "./question-formulator.js";
import { graphUpdateSchema, situationGraphSchema } from "./schema.js";
import {
applyGraphUpdate,
@@ -623,18 +626,25 @@ export function applyValidatedProposal({ situationGraph, proposal }) {
updatedSituationGraph.resolvedNodeIds,
);
if (deterministicSelection?.nodeId) {
if (
deterministicSelection?.status === "selected" &&
deterministicSelection?.nodeId
) {
newActiveUnknownNodeId = deterministicSelection.nodeId;
} else if (deterministicSelection?.status === "ambiguous") {
newActiveUnknownNodeId = null;
}
updatedSituationGraph.activeUnknownNodeId = newActiveUnknownNodeId;
updatedSituationGraph.currentSummary = describeGraph(updatedSituationGraph);
const selectedNode = deterministicSelection?.nodeId
? updatedSituationGraph.nodes.find(
(node) => node.id === deterministicSelection.nodeId,
)
: null;
const selectedNode =
deterministicSelection?.status === "selected" &&
deterministicSelection?.nodeId
? updatedSituationGraph.nodes.find(
(node) => node.id === deterministicSelection.nodeId,
)
: null;
const formulatedQuestion = selectedNode
? formulateQuestion({
node: selectedNode,
@@ -645,20 +655,28 @@ export function applyValidatedProposal({ situationGraph, proposal }) {
.filter(
(value) => typeof value === "string" && value.trim().length > 0,
),
selectionState: deterministicSelection,
},
})
: null;
const finalSelectedQuestion = deterministicSelection
? {
nodeId: deterministicSelection.nodeId,
question:
formulatedQuestion?.question || deterministicSelection.question,
reason: formulatedQuestion?.reason || deterministicSelection.reason,
strategy: formulatedQuestion?.strategy,
investigationStrategy: formulatedQuestion?.investigationStrategy,
}
: null;
const finalSelectedQuestion =
deterministicSelection?.status === "ambiguous"
? {
nodeId: null,
tiedCandidateIds: deterministicSelection.tiedCandidateIds,
...formulateTieResolutionQuestion({ graph: updatedSituationGraph }),
}
: deterministicSelection?.status === "selected"
? {
nodeId: deterministicSelection.nodeId,
question:
formulatedQuestion?.question || deterministicSelection.question,
reason: formulatedQuestion?.reason || deterministicSelection.reason,
strategy: formulatedQuestion?.strategy,
investigationStrategy: formulatedQuestion?.investigationStrategy,
}
: null;
const resultGraphValidation = situationGraphSchema.safeParse(
updatedSituationGraph,
lib/graph/orchestrator.js
+46 -11
View File
@@ -15,6 +15,7 @@ import {
import { buildInitialGraph, describeGraph } from "./builder.js";
import { applyValidatedProposal } from "./apply-proposal.js";
import { buildGraphUpdatePrompt } from "./prompt-builder.js";
import { formulateTieResolutionQuestion } from "./question-formulator.js";
import { parseGraphUpdateProposal } from "./update-proposal.js";
import {
explainUnknownSelection,
@@ -53,6 +54,26 @@ function buildDiagnostics({
};
}
function buildUnknownSelectionDiagnostics(
graph,
resolvedNodeIds = [],
selectedQuestion = null,
) {
const explanation = explainUnknownSelection(graph, resolvedNodeIds);
if (explanation.status === "ambiguous") {
return {
...explanation,
tieResolutionQuestion:
selectedQuestion?.selectionStatus === "ambiguous"
? selectedQuestion.question
: formulateTieResolutionQuestion({ graph }).question,
alphabeticalUsedAsReasoning: false,
};
}
return explanation;
}
function buildUpdateDiagnostics({
promptVersion,
modelName,
@@ -119,14 +140,17 @@ export async function startCase(body) {
});
const currentSummary = describeGraph(initialGraph);
const deterministicSelection = selectActiveUnknownCandidate(
{
...initialGraph,
resolvedNodeIds: [],
},
[],
);
const activeUnknownNodeId =
selectActiveUnknownCandidate(
{
...initialGraph,
resolvedNodeIds: [],
},
[],
)?.nodeId ?? null;
deterministicSelection?.status === "selected"
? deterministicSelection.nodeId
: null;
const situationGraph = makeGraph({
centralStatement: scenario,
@@ -140,9 +164,18 @@ export async function startCase(body) {
situationGraphSchema.parse(situationGraph);
const graphReferenceValidation = validateGraphReferences(situationGraph);
const unknownSelectionExplanation = explainUnknownSelection(
const selectedQuestion =
deterministicSelection?.status === "ambiguous"
? {
id: "q_tie_resolution",
...formulateTieResolutionQuestion({ graph: situationGraph }),
tiedCandidateIds: deterministicSelection.tiedCandidateIds,
}
: (analysis.nextQuestion ?? null);
const unknownSelectionExplanation = buildUnknownSelectionDiagnostics(
situationGraph,
[],
selectedQuestion,
);
if (!graphReferenceValidation.valid) {
return {
@@ -162,7 +195,7 @@ export async function startCase(body) {
return {
success: true,
situationGraph,
selectedQuestion: analysis.nextQuestion ?? null,
selectedQuestion,
diagnostics: buildDiagnostics({
analysis,
graph: situationGraph,
@@ -339,9 +372,10 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
graph: applicationResult.updatedSituationGraph,
graphReferenceValidation: applicationResult.graphReferenceValidation,
selectedQuestion: applicationResult.selectedQuestion,
unknownSelectionExplanation: explainUnknownSelection(
unknownSelectionExplanation: buildUnknownSelectionDiagnostics(
applicationResult.updatedSituationGraph,
applicationResult.updatedSituationGraph.resolvedNodeIds || [],
applicationResult.selectedQuestion,
),
}),
};
@@ -359,9 +393,10 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
graph: situationGraph,
graphReferenceValidation,
selectedQuestion: null,
unknownSelectionExplanation: explainUnknownSelection(
unknownSelectionExplanation: buildUnknownSelectionDiagnostics(
situationGraph,
situationGraph.resolvedNodeIds || [],
null,
),
}),
};
lib/graph/question-formulator.js
+132 -23
View File
@@ -11,6 +11,13 @@ function sentenceCase(value) {
return trimmed.charAt(0).toLowerCase() + trimmed.slice(1);
}
function stripTrailingPunctuation(value) {
return String(value || "")
.trim()
.replace(/[.?!:;]+$/g, "")
.trim();
}
function buildNodeMap(graph) {
return new Map((graph?.nodes || []).map((node) => [node.id, node]));
}
@@ -52,8 +59,8 @@ function collectResolvedContextValues(graph) {
function extractMeaning(node) {
const raw = `${node?.label || ""} ${node?.description || ""}`.trim();
let meaning = String(
node?.label || node?.description || "this uncertainty",
let meaning = stripTrailingPunctuation(
String(node?.label || node?.description || "this uncertainty"),
).trim();
const lowered = normaliseText(raw);
@@ -79,6 +86,84 @@ function extractMeaning(node) {
return sentenceCase(meaning);
}
function isDefinitionLikeUnknown(nodeText, text) {
return (
/\b(define|definition|meaning|term|terminology)\b/.test(nodeText) ||
(/\bdefinition\b/.test(text) && /\bdisagreement\b/.test(text)) ||
(/\b(define|definition|meaning|term|terminology)\b/.test(text) &&
/\b(unclear|ambiguous|inconsistent|undefined|used inconsistently)\b/.test(
text,
))
);
}
function isClaimLikeUnknown(node, text) {
return (
node?.kind === "reported_claim" ||
node?.kind === "conclusion" ||
/\b(claim|assertion|true|false|correct|incorrect|happened|happening)\b/.test(
text,
) ||
/^whether\b/i.test(String(node?.label || "").trim())
);
}
function sanitizeQuestionText(question) {
return String(question || "")
.replace(/\)\.\s+/g, ") ")
.replace(/\s+/g, " ")
.trim();
}
function buildNeutralClarificationQuestion(meaning) {
return `What would clarify ${stripTrailingPunctuation(meaning)} in this situation?`;
}
function buildEvidenceFallbackQuestion(meaning) {
return `What evidence would confirm or rule out ${stripTrailingPunctuation(meaning)}?`;
}
function detectContradictionContext(graph) {
const central = stripTrailingPunctuation(
graph?.centralStatement || "this situation",
);
const contradictionNode = (graph?.nodes || []).find((node) => {
const text = normaliseText(`${node.label} ${node.description}`);
return (
node.kind === "relationship" &&
/\b(contradiction|conflict|inconsistent|mismatch|divergent|opposing)\b/.test(
text,
)
);
});
return {
centralStatement: central,
contradictionLabel: stripTrailingPunctuation(
contradictionNode?.label || "",
),
};
}
export function formulateTieResolutionQuestion({ graph }) {
const { centralStatement, contradictionLabel } =
detectContradictionContext(graph);
const focus =
centralStatement || contradictionLabel || "these conflicting signals";
const question = sanitizeQuestionText(
`What changed during the period that could explain why ${focus}?`,
);
return {
question,
reason:
"Formulated to distinguish between tied unresolved explanations without prematurely choosing one branch.",
strategy: null,
investigationStrategy: null,
selectionStatus: "ambiguous",
};
}
function extractActionPhrase(texts) {
for (const text of texts) {
const value = String(text || "").trim();
@@ -224,8 +309,7 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) {
});
}
const hasDefinitionLanguage =
/\b(define|definition|meaning|term|terminology)\b/.test(text);
const hasDefinitionLanguage = isDefinitionLikeUnknown(nodeText, text);
const hasPrimaryDefinitionLanguage =
/\b(define|definition|meaning|term|terminology)\b/.test(nodeText);
const hasCriteriaLanguage =
@@ -243,9 +327,7 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) {
);
const hasEvidenceLanguage =
/\b(evidence|proof|validate|validation|signal|demand)\b/.test(text) ||
node?.kind === "reported_claim" ||
node?.kind === "conclusion" ||
/\b(claim|assertion|true|false)\b/.test(text);
isClaimLikeUnknown(node, text);
const hasContradictionLanguage =
/\b(contradiction|contradict|conflict|inconsistent|inconsistency|disagree|mismatch)\b/.test(
`${text} ${relatedText}`,
@@ -323,16 +405,7 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) {
});
}
return buildInvestigationStrategy({
key: "definition",
reason:
"Selected as the deterministic fallback because clarifying the exact meaning of the unknown is the narrowest first step.",
node,
graph,
relatedNodes,
meaning,
actionPhrase,
});
return null;
}
function buildQuestionFromStrategy(strategy) {
@@ -344,7 +417,7 @@ function buildQuestionFromStrategy(strategy) {
case "definition":
return `What does ${strategy.meaning} mean in this situation?`;
case "evidence_gathering":
return `What evidence would show whether ${strategy.meaning} is true?`;
return `What evidence would clarify ${stripTrailingPunctuation(strategy.meaning)}?`;
case "baseline_reconstruction":
return `What was the comparable state before ${strategy.meaning}?`;
case "contradiction_resolution":
@@ -382,6 +455,9 @@ function validateFormulatedQuestion(question, meaning) {
if (/^how should uncertainty regarding\b/i.test(trimmed)) return false;
if (/^what would resolve uncertainty regarding\b/i.test(trimmed))
return false;
if (/\)\.\s+[A-Z]/.test(trimmed)) return false;
if (/\bis true\?$/i.test(trimmed) && !/^whether\b/i.test(meaning))
return false;
if (
/\bprice|pricing|price point\b/i.test(trimmed) &&
!/\bprice\b/i.test(meaning)
@@ -399,22 +475,55 @@ function validateFormulatedQuestion(question, meaning) {
}
export function formulateQuestion({ node, graph, context = {} }) {
if (context.selectionState?.status === "ambiguous") {
return formulateTieResolutionQuestion({ graph });
}
const investigationStrategy = selectInvestigationStrategy({
node,
graph,
context,
});
let question = buildQuestionFromStrategy(investigationStrategy);
let question = investigationStrategy
? buildQuestionFromStrategy(investigationStrategy)
: buildNeutralClarificationQuestion(extractMeaning(node));
if (!validateFormulatedQuestion(question, investigationStrategy.meaning)) {
question = `What evidence would resolve whether ${investigationStrategy.meaning} is true?`;
question = sanitizeQuestionText(question);
const fallbackMeaning = extractMeaning(node);
if (
!validateFormulatedQuestion(
question,
investigationStrategy?.meaning || fallbackMeaning,
)
) {
question = sanitizeQuestionText(
investigationStrategy &&
isClaimLikeUnknown(
node,
normaliseText(
collectRelatedNodes(node, graph)
.map(
(relatedNode) =>
`${relatedNode.label} ${relatedNode.description}`,
)
.concat([node?.label, node?.description])
.filter(Boolean)
.join(" "),
),
)
? buildEvidenceFallbackQuestion(fallbackMeaning)
: buildNeutralClarificationQuestion(fallbackMeaning),
);
}
return {
question,
reason: `Formulated from graph context using the ${investigationStrategy.key} investigation strategy.`,
strategy: investigationStrategy.key,
reason: investigationStrategy
? `Formulated from graph context using the ${investigationStrategy.key} investigation strategy.`
: "Formulated as a neutral clarification question because no narrower investigation strategy clearly applied.",
strategy: investigationStrategy?.key ?? null,
investigationStrategy,
};
}
lib/graph/utils.js
+200 -62
View File
@@ -216,6 +216,119 @@ function buildScoreContributions(
return contributions;
}
function getMeaningfulSemanticContributions(contributions = []) {
return contributions
.filter(
(contribution) =>
contribution.rule !== "downstream_dependencies" &&
contribution.rule !== "unresolved_prerequisite_penalty" &&
contribution.delta !== 0,
)
.map((contribution) => ({
rule: contribution.rule,
delta: contribution.delta,
}));
}
function buildCandidateDisplayOrder(candidates) {
return [...candidates].sort((a, b) => {
if (b.score !== a.score) return b.score - a.score;
if (b.downstreamCount !== a.downstreamCount) {
return b.downstreamCount - a.downstreamCount;
}
if (a.unresolvedParentUnknownCount !== b.unresolvedParentUnknownCount) {
return a.unresolvedParentUnknownCount - b.unresolvedParentUnknownCount;
}
return a.label.localeCompare(b.label);
});
}
function semanticSignature(candidate) {
return JSON.stringify(
getMeaningfulSemanticContributions(candidate.contributions),
);
}
function classifyCandidateOrdering(candidates) {
const displayOrder = buildCandidateDisplayOrder(candidates);
const best = displayOrder[0] ?? null;
if (!best) {
return {
displayOrder,
best: null,
leadingCandidates: [],
status: "no_candidates",
tieType: "none",
usedAlphabeticalOrdering: false,
reason: "No unresolved unknown candidates remain.",
};
}
const topScoreCandidates = displayOrder.filter(
(candidate) => candidate.score === best.score,
);
if (topScoreCandidates.length === 1) {
return {
displayOrder,
best,
leadingCandidates: [best],
status: "selected",
tieType: "none",
usedAlphabeticalOrdering: false,
reason: `Clear winner by total score (${best.score}).`,
};
}
const topStructuralCandidates = topScoreCandidates.filter(
(candidate) =>
candidate.downstreamCount === best.downstreamCount &&
candidate.unresolvedParentUnknownCount ===
best.unresolvedParentUnknownCount,
);
if (topStructuralCandidates.length === 1) {
return {
displayOrder,
best,
leadingCandidates: [best],
status: "selected",
tieType: "structural_tie",
usedAlphabeticalOrdering: false,
reason:
"Score tie was resolved by downstream dependency count or prerequisite ordering.",
};
}
const topSemanticSignature = semanticSignature(best);
const semanticPeers = topStructuralCandidates.filter(
(candidate) => semanticSignature(candidate) === topSemanticSignature,
);
if (semanticPeers.length !== topStructuralCandidates.length) {
return {
displayOrder,
best: null,
leadingCandidates: topStructuralCandidates,
status: "ambiguous",
tieType: "semantic_tie",
usedAlphabeticalOrdering: false,
reason:
"Leading candidates remain tied after score and structural checks, but differ in semantic contribution patterns.",
};
}
return {
displayOrder,
best: null,
leadingCandidates: topStructuralCandidates,
status: "ambiguous",
tieType: "complete_unresolved_tie",
usedAlphabeticalOrdering: false,
reason: "No justified distinction between leading unknowns.",
};
}
export function scoreUnknownCandidate(graph, node, resolvedNodeIds = []) {
const text = collectNodeText(node);
const matches = classifyUnknownPriority(text);
@@ -490,21 +603,36 @@ export function selectActiveUnknownCandidate(graph, resolvedNodeIds) {
...scoreUnknownCandidate(graph, node, resolvedNodeIds),
}));
scoredCandidates.sort((a, b) => {
if (b.score !== a.score) return b.score - a.score;
if (b.downstreamCount !== a.downstreamCount) {
return b.downstreamCount - a.downstreamCount;
}
if (a.unresolvedParentUnknownCount !== b.unresolvedParentUnknownCount) {
return a.unresolvedParentUnknownCount - b.unresolvedParentUnknownCount;
}
return a.node.label.localeCompare(b.node.label);
});
const selection = classifyCandidateOrdering(
scoredCandidates.map(({ node, ...candidate }) => ({
...candidate,
node,
})),
);
const best = scoredCandidates[0];
if (selection.status === "ambiguous") {
return {
selectedNode: null,
status: "ambiguous",
tieType: selection.tieType,
tiedCandidateIds: selection.leadingCandidates.map(
(candidate) => candidate.nodeId,
),
displayOrder: selection.displayOrder.map((candidate) => candidate.nodeId),
reason: selection.reason,
};
}
const best = selection.best;
if (!best) return null;
return {
selectedNode: {
nodeId: best.node.id,
label: best.node.label,
},
status: "selected",
tieType: selection.tieType,
nodeId: best.node.id,
label: best.node.label,
score: best.score,
@@ -522,7 +650,10 @@ export function explainUnknownSelection(graph, resolvedNodeIds = []) {
return {
selectedNodeId: null,
selectedNodeLabel: null,
status: "no_candidates",
tieType: "none",
resolvedNodeIds: [...resolvedNodeIds],
tiedCandidateIds: [],
candidates: [],
competitors: [],
tieBreakOrder: [
@@ -543,68 +674,75 @@ export function explainUnknownSelection(graph, resolvedNodeIds = []) {
...scoreUnknownCandidate(graph, node, resolvedNodeIds),
}));
candidates.sort((a, b) => {
if (b.score !== a.score) return b.score - a.score;
if (b.downstreamCount !== a.downstreamCount) {
return b.downstreamCount - a.downstreamCount;
}
if (a.unresolvedParentUnknownCount !== b.unresolvedParentUnknownCount) {
return a.unresolvedParentUnknownCount - b.unresolvedParentUnknownCount;
}
return a.label.localeCompare(b.label);
});
const selected = candidates[0];
const competitors = candidates.slice(1).map((candidate) => ({
nodeId: candidate.nodeId,
label: candidate.label,
score: candidate.score,
downstreamCount: candidate.downstreamCount,
unresolvedParentUnknownCount: candidate.unresolvedParentUnknownCount,
matches: candidate.matches,
contributions: candidate.contributions,
outrankedBy: {
scoreDelta: selected.score - candidate.score,
downstreamDelta: selected.downstreamCount - candidate.downstreamCount,
unresolvedPrerequisiteDelta:
candidate.unresolvedParentUnknownCount -
selected.unresolvedParentUnknownCount,
labelOrderWinner:
selected.score === candidate.score &&
selected.downstreamCount === candidate.downstreamCount &&
selected.unresolvedParentUnknownCount ===
candidate.unresolvedParentUnknownCount
? selected.label.localeCompare(candidate.label) <= 0
? selected.label
: candidate.label
: null,
},
}));
const selection = classifyCandidateOrdering(candidates);
const orderedCandidates = selection.displayOrder;
const selected = selection.best;
const competitors = orderedCandidates
.filter((candidate) => candidate.nodeId !== selected?.nodeId)
.map((candidate) => ({
nodeId: candidate.nodeId,
label: candidate.label,
score: candidate.score,
downstreamCount: candidate.downstreamCount,
unresolvedParentUnknownCount: candidate.unresolvedParentUnknownCount,
matches: candidate.matches,
contributions: candidate.contributions,
outrankedBy: {
scoreDelta: (selected?.score ?? candidate.score) - candidate.score,
downstreamDelta:
(selected?.downstreamCount ?? candidate.downstreamCount) -
candidate.downstreamCount,
unresolvedPrerequisiteDelta:
candidate.unresolvedParentUnknownCount -
(selected?.unresolvedParentUnknownCount ??
candidate.unresolvedParentUnknownCount),
labelOrderWinner:
selected &&
selected.score === candidate.score &&
selected.downstreamCount === candidate.downstreamCount &&
selected.unresolvedParentUnknownCount ===
candidate.unresolvedParentUnknownCount
? selected.label.localeCompare(candidate.label) <= 0
? selected.label
: candidate.label
: null,
},
}));
return {
selectedNodeId: selected.nodeId,
selectedNodeLabel: selected.label,
selectedNodeId: selected?.nodeId ?? null,
selectedNodeLabel: selected?.label ?? null,
status: selection.status,
tieType: selection.tieType,
resolvedNodeIds: [...resolvedNodeIds],
tiedCandidateIds: selection.leadingCandidates.map(
(candidate) => candidate.nodeId,
),
tieBreakOrder: [
"score_desc",
"downstreamCount_desc",
"unresolvedParentUnknownCount_asc",
"label_asc",
],
candidates,
selected: {
nodeId: selected.nodeId,
label: selected.label,
score: selected.score,
downstreamCount: selected.downstreamCount,
unresolvedParentUnknownCount: selected.unresolvedParentUnknownCount,
matches: selected.matches,
contributions: selected.contributions,
},
alphabeticalUsedAsReasoning: false,
candidates: orderedCandidates,
selected: selected
? {
nodeId: selected.nodeId,
label: selected.label,
score: selected.score,
downstreamCount: selected.downstreamCount,
unresolvedParentUnknownCount: selected.unresolvedParentUnknownCount,
matches: selected.matches,
contributions: selected.contributions,
}
: null,
competitors,
summary: {
candidateCount: candidates.length,
selectedReason: `highest_score=${selected.score}; downstream=${selected.downstreamCount}; unresolved_prerequisites=${selected.unresolvedParentUnknownCount}`,
candidateCount: orderedCandidates.length,
selectedReason: selected
? `highest_score=${selected.score}; downstream=${selected.downstreamCount}; unresolved_prerequisites=${selected.unresolvedParentUnknownCount}`
: selection.reason,
},
};
}