robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: handle unjustified unknown selection ties

Browse Source
This commit is contained in:
robbond
2026-08-02 16:09:00 +01:00
parent a1f6d0c2b9
commit 51ce356218
8 changed files with 731 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/graph/apply-proposal.js
+35 -17
View File
@@ -1,5 +1,8 @@
import { describeGraph } from "./builder.js";
import { formulateQuestion } from "./question-formulator.js";
import {
formulateQuestion,
formulateTieResolutionQuestion,
} from "./question-formulator.js";
import { graphUpdateSchema, situationGraphSchema } from "./schema.js";
import {
applyGraphUpdate,
@@ -623,18 +626,25 @@ export function applyValidatedProposal({ situationGraph, proposal }) {
updatedSituationGraph.resolvedNodeIds,
);
if (deterministicSelection?.nodeId) {
if (
deterministicSelection?.status === "selected" &&
deterministicSelection?.nodeId
) {
newActiveUnknownNodeId = deterministicSelection.nodeId;
} else if (deterministicSelection?.status === "ambiguous") {
newActiveUnknownNodeId = null;
}
updatedSituationGraph.activeUnknownNodeId = newActiveUnknownNodeId;
updatedSituationGraph.currentSummary = describeGraph(updatedSituationGraph);
const selectedNode = deterministicSelection?.nodeId
? updatedSituationGraph.nodes.find(
(node) => node.id === deterministicSelection.nodeId,
)
: null;
const selectedNode =
deterministicSelection?.status === "selected" &&
deterministicSelection?.nodeId
? updatedSituationGraph.nodes.find(
(node) => node.id === deterministicSelection.nodeId,
)
: null;
const formulatedQuestion = selectedNode
? formulateQuestion({
node: selectedNode,
@@ -645,20 +655,28 @@ export function applyValidatedProposal({ situationGraph, proposal }) {
.filter(
(value) => typeof value === "string" && value.trim().length > 0,
),
selectionState: deterministicSelection,
},
})
: null;
const finalSelectedQuestion = deterministicSelection
? {
nodeId: deterministicSelection.nodeId,
question:
formulatedQuestion?.question || deterministicSelection.question,
reason: formulatedQuestion?.reason || deterministicSelection.reason,
strategy: formulatedQuestion?.strategy,
investigationStrategy: formulatedQuestion?.investigationStrategy,
}
: null;
const finalSelectedQuestion =
deterministicSelection?.status === "ambiguous"
? {
nodeId: null,
tiedCandidateIds: deterministicSelection.tiedCandidateIds,
...formulateTieResolutionQuestion({ graph: updatedSituationGraph }),
}
: deterministicSelection?.status === "selected"
? {
nodeId: deterministicSelection.nodeId,
question:
formulatedQuestion?.question || deterministicSelection.question,
reason: formulatedQuestion?.reason || deterministicSelection.reason,
strategy: formulatedQuestion?.strategy,
investigationStrategy: formulatedQuestion?.investigationStrategy,
}
: null;
const resultGraphValidation = situationGraphSchema.safeParse(
updatedSituationGraph,
lib/graph/orchestrator.js
+46 -11
View File
@@ -15,6 +15,7 @@ import {
import { buildInitialGraph, describeGraph } from "./builder.js";
import { applyValidatedProposal } from "./apply-proposal.js";
import { buildGraphUpdatePrompt } from "./prompt-builder.js";
import { formulateTieResolutionQuestion } from "./question-formulator.js";
import { parseGraphUpdateProposal } from "./update-proposal.js";
import {
explainUnknownSelection,
@@ -53,6 +54,26 @@ function buildDiagnostics({
};
}
function buildUnknownSelectionDiagnostics(
graph,
resolvedNodeIds = [],
selectedQuestion = null,
) {
const explanation = explainUnknownSelection(graph, resolvedNodeIds);
if (explanation.status === "ambiguous") {
return {
...explanation,
tieResolutionQuestion:
selectedQuestion?.selectionStatus === "ambiguous"
? selectedQuestion.question
: formulateTieResolutionQuestion({ graph }).question,
alphabeticalUsedAsReasoning: false,
};
}
return explanation;
}
function buildUpdateDiagnostics({
promptVersion,
modelName,
@@ -119,14 +140,17 @@ export async function startCase(body) {
});
const currentSummary = describeGraph(initialGraph);
const deterministicSelection = selectActiveUnknownCandidate(
{
...initialGraph,
resolvedNodeIds: [],
},
[],
);
const activeUnknownNodeId =
selectActiveUnknownCandidate(
{
...initialGraph,
resolvedNodeIds: [],
},
[],
)?.nodeId ?? null;
deterministicSelection?.status === "selected"
? deterministicSelection.nodeId
: null;
const situationGraph = makeGraph({
centralStatement: scenario,
@@ -140,9 +164,18 @@ export async function startCase(body) {
situationGraphSchema.parse(situationGraph);
const graphReferenceValidation = validateGraphReferences(situationGraph);
const unknownSelectionExplanation = explainUnknownSelection(
const selectedQuestion =
deterministicSelection?.status === "ambiguous"
? {
id: "q_tie_resolution",
...formulateTieResolutionQuestion({ graph: situationGraph }),
tiedCandidateIds: deterministicSelection.tiedCandidateIds,
}
: (analysis.nextQuestion ?? null);
const unknownSelectionExplanation = buildUnknownSelectionDiagnostics(
situationGraph,
[],
selectedQuestion,
);
if (!graphReferenceValidation.valid) {
return {
@@ -162,7 +195,7 @@ export async function startCase(body) {
return {
success: true,
situationGraph,
selectedQuestion: analysis.nextQuestion ?? null,
selectedQuestion,
diagnostics: buildDiagnostics({
analysis,
graph: situationGraph,
@@ -339,9 +372,10 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
graph: applicationResult.updatedSituationGraph,
graphReferenceValidation: applicationResult.graphReferenceValidation,
selectedQuestion: applicationResult.selectedQuestion,
unknownSelectionExplanation: explainUnknownSelection(
unknownSelectionExplanation: buildUnknownSelectionDiagnostics(
applicationResult.updatedSituationGraph,
applicationResult.updatedSituationGraph.resolvedNodeIds || [],
applicationResult.selectedQuestion,
),
}),
};
@@ -359,9 +393,10 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
graph: situationGraph,
graphReferenceValidation,
selectedQuestion: null,
unknownSelectionExplanation: explainUnknownSelection(
unknownSelectionExplanation: buildUnknownSelectionDiagnostics(
situationGraph,
situationGraph.resolvedNodeIds || [],
null,
),
}),
};
lib/graph/question-formulator.js
+132 -23
View File
@@ -11,6 +11,13 @@ function sentenceCase(value) {
return trimmed.charAt(0).toLowerCase() + trimmed.slice(1);
}
function stripTrailingPunctuation(value) {
return String(value || "")
.trim()
.replace(/[.?!:;]+$/g, "")
.trim();
}
function buildNodeMap(graph) {
return new Map((graph?.nodes || []).map((node) => [node.id, node]));
}
@@ -52,8 +59,8 @@ function collectResolvedContextValues(graph) {
function extractMeaning(node) {
const raw = `${node?.label || ""} ${node?.description || ""}`.trim();
let meaning = String(
node?.label || node?.description || "this uncertainty",
let meaning = stripTrailingPunctuation(
String(node?.label || node?.description || "this uncertainty"),
).trim();
const lowered = normaliseText(raw);
@@ -79,6 +86,84 @@ function extractMeaning(node) {
return sentenceCase(meaning);
}
function isDefinitionLikeUnknown(nodeText, text) {
return (
/\b(define|definition|meaning|term|terminology)\b/.test(nodeText) ||
(/\bdefinition\b/.test(text) && /\bdisagreement\b/.test(text)) ||
(/\b(define|definition|meaning|term|terminology)\b/.test(text) &&
/\b(unclear|ambiguous|inconsistent|undefined|used inconsistently)\b/.test(
text,
))
);
}
function isClaimLikeUnknown(node, text) {
return (
node?.kind === "reported_claim" ||
node?.kind === "conclusion" ||
/\b(claim|assertion|true|false|correct|incorrect|happened|happening)\b/.test(
text,
) ||
/^whether\b/i.test(String(node?.label || "").trim())
);
}
function sanitizeQuestionText(question) {
return String(question || "")
.replace(/\)\.\s+/g, ") ")
.replace(/\s+/g, " ")
.trim();
}
function buildNeutralClarificationQuestion(meaning) {
return `What would clarify ${stripTrailingPunctuation(meaning)} in this situation?`;
}
function buildEvidenceFallbackQuestion(meaning) {
return `What evidence would confirm or rule out ${stripTrailingPunctuation(meaning)}?`;
}
function detectContradictionContext(graph) {
const central = stripTrailingPunctuation(
graph?.centralStatement || "this situation",
);
const contradictionNode = (graph?.nodes || []).find((node) => {
const text = normaliseText(`${node.label} ${node.description}`);
return (
node.kind === "relationship" &&
/\b(contradiction|conflict|inconsistent|mismatch|divergent|opposing)\b/.test(
text,
)
);
});
return {
centralStatement: central,
contradictionLabel: stripTrailingPunctuation(
contradictionNode?.label || "",
),
};
}
export function formulateTieResolutionQuestion({ graph }) {
const { centralStatement, contradictionLabel } =
detectContradictionContext(graph);
const focus =
centralStatement || contradictionLabel || "these conflicting signals";
const question = sanitizeQuestionText(
`What changed during the period that could explain why ${focus}?`,
);
return {
question,
reason:
"Formulated to distinguish between tied unresolved explanations without prematurely choosing one branch.",
strategy: null,
investigationStrategy: null,
selectionStatus: "ambiguous",
};
}
function extractActionPhrase(texts) {
for (const text of texts) {
const value = String(text || "").trim();
@@ -224,8 +309,7 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) {
});
}
const hasDefinitionLanguage =
/\b(define|definition|meaning|term|terminology)\b/.test(text);
const hasDefinitionLanguage = isDefinitionLikeUnknown(nodeText, text);
const hasPrimaryDefinitionLanguage =
/\b(define|definition|meaning|term|terminology)\b/.test(nodeText);
const hasCriteriaLanguage =
@@ -243,9 +327,7 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) {
);
const hasEvidenceLanguage =
/\b(evidence|proof|validate|validation|signal|demand)\b/.test(text) ||
node?.kind === "reported_claim" ||
node?.kind === "conclusion" ||
/\b(claim|assertion|true|false)\b/.test(text);
isClaimLikeUnknown(node, text);
const hasContradictionLanguage =
/\b(contradiction|contradict|conflict|inconsistent|inconsistency|disagree|mismatch)\b/.test(
`${text} ${relatedText}`,
@@ -323,16 +405,7 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) {
});
}
return buildInvestigationStrategy({
key: "definition",
reason:
"Selected as the deterministic fallback because clarifying the exact meaning of the unknown is the narrowest first step.",
node,
graph,
relatedNodes,
meaning,
actionPhrase,
});
return null;
}
function buildQuestionFromStrategy(strategy) {
@@ -344,7 +417,7 @@ function buildQuestionFromStrategy(strategy) {
case "definition":
return `What does ${strategy.meaning} mean in this situation?`;
case "evidence_gathering":
return `What evidence would show whether ${strategy.meaning} is true?`;
return `What evidence would clarify ${stripTrailingPunctuation(strategy.meaning)}?`;
case "baseline_reconstruction":
return `What was the comparable state before ${strategy.meaning}?`;
case "contradiction_resolution":
@@ -382,6 +455,9 @@ function validateFormulatedQuestion(question, meaning) {
if (/^how should uncertainty regarding\b/i.test(trimmed)) return false;
if (/^what would resolve uncertainty regarding\b/i.test(trimmed))
return false;
if (/\)\.\s+[A-Z]/.test(trimmed)) return false;
if (/\bis true\?$/i.test(trimmed) && !/^whether\b/i.test(meaning))
return false;
if (
/\bprice|pricing|price point\b/i.test(trimmed) &&
!/\bprice\b/i.test(meaning)
@@ -399,22 +475,55 @@ function validateFormulatedQuestion(question, meaning) {
}
export function formulateQuestion({ node, graph, context = {} }) {
if (context.selectionState?.status === "ambiguous") {
return formulateTieResolutionQuestion({ graph });
}
const investigationStrategy = selectInvestigationStrategy({
node,
graph,
context,
});
let question = buildQuestionFromStrategy(investigationStrategy);
let question = investigationStrategy
? buildQuestionFromStrategy(investigationStrategy)
: buildNeutralClarificationQuestion(extractMeaning(node));
if (!validateFormulatedQuestion(question, investigationStrategy.meaning)) {
question = `What evidence would resolve whether ${investigationStrategy.meaning} is true?`;
question = sanitizeQuestionText(question);
const fallbackMeaning = extractMeaning(node);
if (
!validateFormulatedQuestion(
question,
investigationStrategy?.meaning || fallbackMeaning,
)
) {
question = sanitizeQuestionText(
investigationStrategy &&
isClaimLikeUnknown(
node,
normaliseText(
collectRelatedNodes(node, graph)
.map(
(relatedNode) =>
`${relatedNode.label} ${relatedNode.description}`,
)
.concat([node?.label, node?.description])
.filter(Boolean)
.join(" "),
),
)
? buildEvidenceFallbackQuestion(fallbackMeaning)
: buildNeutralClarificationQuestion(fallbackMeaning),
);
}
return {
question,
reason: `Formulated from graph context using the ${investigationStrategy.key} investigation strategy.`,
strategy: investigationStrategy.key,
reason: investigationStrategy
? `Formulated from graph context using the ${investigationStrategy.key} investigation strategy.`
: "Formulated as a neutral clarification question because no narrower investigation strategy clearly applied.",
strategy: investigationStrategy?.key ?? null,
investigationStrategy,
};
}
lib/graph/utils.js
+200 -62
View File
@@ -216,6 +216,119 @@ function buildScoreContributions(
return contributions;
}
function getMeaningfulSemanticContributions(contributions = []) {
return contributions
.filter(
(contribution) =>
contribution.rule !== "downstream_dependencies" &&
contribution.rule !== "unresolved_prerequisite_penalty" &&
contribution.delta !== 0,
)
.map((contribution) => ({
rule: contribution.rule,
delta: contribution.delta,
}));
}
function buildCandidateDisplayOrder(candidates) {
return [...candidates].sort((a, b) => {
if (b.score !== a.score) return b.score - a.score;
if (b.downstreamCount !== a.downstreamCount) {
return b.downstreamCount - a.downstreamCount;
}
if (a.unresolvedParentUnknownCount !== b.unresolvedParentUnknownCount) {
return a.unresolvedParentUnknownCount - b.unresolvedParentUnknownCount;
}
return a.label.localeCompare(b.label);
});
}
function semanticSignature(candidate) {
return JSON.stringify(
getMeaningfulSemanticContributions(candidate.contributions),
);
}
function classifyCandidateOrdering(candidates) {
const displayOrder = buildCandidateDisplayOrder(candidates);
const best = displayOrder[0] ?? null;
if (!best) {
return {
displayOrder,
best: null,
leadingCandidates: [],
status: "no_candidates",
tieType: "none",
usedAlphabeticalOrdering: false,
reason: "No unresolved unknown candidates remain.",
};
}
const topScoreCandidates = displayOrder.filter(
(candidate) => candidate.score === best.score,
);
if (topScoreCandidates.length === 1) {
return {
displayOrder,
best,
leadingCandidates: [best],
status: "selected",
tieType: "none",
usedAlphabeticalOrdering: false,
reason: `Clear winner by total score (${best.score}).`,
};
}
const topStructuralCandidates = topScoreCandidates.filter(
(candidate) =>
candidate.downstreamCount === best.downstreamCount &&
candidate.unresolvedParentUnknownCount ===
best.unresolvedParentUnknownCount,
);
if (topStructuralCandidates.length === 1) {
return {
displayOrder,
best,
leadingCandidates: [best],
status: "selected",
tieType: "structural_tie",
usedAlphabeticalOrdering: false,
reason:
"Score tie was resolved by downstream dependency count or prerequisite ordering.",
};
}
const topSemanticSignature = semanticSignature(best);
const semanticPeers = topStructuralCandidates.filter(
(candidate) => semanticSignature(candidate) === topSemanticSignature,
);
if (semanticPeers.length !== topStructuralCandidates.length) {
return {
displayOrder,
best: null,
leadingCandidates: topStructuralCandidates,
status: "ambiguous",
tieType: "semantic_tie",
usedAlphabeticalOrdering: false,
reason:
"Leading candidates remain tied after score and structural checks, but differ in semantic contribution patterns.",
};
}
return {
displayOrder,
best: null,
leadingCandidates: topStructuralCandidates,
status: "ambiguous",
tieType: "complete_unresolved_tie",
usedAlphabeticalOrdering: false,
reason: "No justified distinction between leading unknowns.",
};
}
export function scoreUnknownCandidate(graph, node, resolvedNodeIds = []) {
const text = collectNodeText(node);
const matches = classifyUnknownPriority(text);
@@ -490,21 +603,36 @@ export function selectActiveUnknownCandidate(graph, resolvedNodeIds) {
...scoreUnknownCandidate(graph, node, resolvedNodeIds),
}));
scoredCandidates.sort((a, b) => {
if (b.score !== a.score) return b.score - a.score;
if (b.downstreamCount !== a.downstreamCount) {
return b.downstreamCount - a.downstreamCount;
}
if (a.unresolvedParentUnknownCount !== b.unresolvedParentUnknownCount) {
return a.unresolvedParentUnknownCount - b.unresolvedParentUnknownCount;
}
return a.node.label.localeCompare(b.node.label);
});
const selection = classifyCandidateOrdering(
scoredCandidates.map(({ node, ...candidate }) => ({
...candidate,
node,
})),
);
const best = scoredCandidates[0];
if (selection.status === "ambiguous") {
return {
selectedNode: null,
status: "ambiguous",
tieType: selection.tieType,
tiedCandidateIds: selection.leadingCandidates.map(
(candidate) => candidate.nodeId,
),
displayOrder: selection.displayOrder.map((candidate) => candidate.nodeId),
reason: selection.reason,
};
}
const best = selection.best;
if (!best) return null;
return {
selectedNode: {
nodeId: best.node.id,
label: best.node.label,
},
status: "selected",
tieType: selection.tieType,
nodeId: best.node.id,
label: best.node.label,
score: best.score,
@@ -522,7 +650,10 @@ export function explainUnknownSelection(graph, resolvedNodeIds = []) {
return {
selectedNodeId: null,
selectedNodeLabel: null,
status: "no_candidates",
tieType: "none",
resolvedNodeIds: [...resolvedNodeIds],
tiedCandidateIds: [],
candidates: [],
competitors: [],
tieBreakOrder: [
@@ -543,68 +674,75 @@ export function explainUnknownSelection(graph, resolvedNodeIds = []) {
...scoreUnknownCandidate(graph, node, resolvedNodeIds),
}));
candidates.sort((a, b) => {
if (b.score !== a.score) return b.score - a.score;
if (b.downstreamCount !== a.downstreamCount) {
return b.downstreamCount - a.downstreamCount;
}
if (a.unresolvedParentUnknownCount !== b.unresolvedParentUnknownCount) {
return a.unresolvedParentUnknownCount - b.unresolvedParentUnknownCount;
}
return a.label.localeCompare(b.label);
});
const selected = candidates[0];
const competitors = candidates.slice(1).map((candidate) => ({
nodeId: candidate.nodeId,
label: candidate.label,
score: candidate.score,
downstreamCount: candidate.downstreamCount,
unresolvedParentUnknownCount: candidate.unresolvedParentUnknownCount,
matches: candidate.matches,
contributions: candidate.contributions,
outrankedBy: {
scoreDelta: selected.score - candidate.score,
downstreamDelta: selected.downstreamCount - candidate.downstreamCount,
unresolvedPrerequisiteDelta:
candidate.unresolvedParentUnknownCount -
selected.unresolvedParentUnknownCount,
labelOrderWinner:
selected.score === candidate.score &&
selected.downstreamCount === candidate.downstreamCount &&
selected.unresolvedParentUnknownCount ===
candidate.unresolvedParentUnknownCount
? selected.label.localeCompare(candidate.label) <= 0
? selected.label
: candidate.label
: null,
},
}));
const selection = classifyCandidateOrdering(candidates);
const orderedCandidates = selection.displayOrder;
const selected = selection.best;
const competitors = orderedCandidates
.filter((candidate) => candidate.nodeId !== selected?.nodeId)
.map((candidate) => ({
nodeId: candidate.nodeId,
label: candidate.label,
score: candidate.score,
downstreamCount: candidate.downstreamCount,
unresolvedParentUnknownCount: candidate.unresolvedParentUnknownCount,
matches: candidate.matches,
contributions: candidate.contributions,
outrankedBy: {
scoreDelta: (selected?.score ?? candidate.score) - candidate.score,
downstreamDelta:
(selected?.downstreamCount ?? candidate.downstreamCount) -
candidate.downstreamCount,
unresolvedPrerequisiteDelta:
candidate.unresolvedParentUnknownCount -
(selected?.unresolvedParentUnknownCount ??
candidate.unresolvedParentUnknownCount),
labelOrderWinner:
selected &&
selected.score === candidate.score &&
selected.downstreamCount === candidate.downstreamCount &&
selected.unresolvedParentUnknownCount ===
candidate.unresolvedParentUnknownCount
? selected.label.localeCompare(candidate.label) <= 0
? selected.label
: candidate.label
: null,
},
}));
return {
selectedNodeId: selected.nodeId,
selectedNodeLabel: selected.label,
selectedNodeId: selected?.nodeId ?? null,
selectedNodeLabel: selected?.label ?? null,
status: selection.status,
tieType: selection.tieType,
resolvedNodeIds: [...resolvedNodeIds],
tiedCandidateIds: selection.leadingCandidates.map(
(candidate) => candidate.nodeId,
),
tieBreakOrder: [
"score_desc",
"downstreamCount_desc",
"unresolvedParentUnknownCount_asc",
"label_asc",
],
candidates,
selected: {
nodeId: selected.nodeId,
label: selected.label,
score: selected.score,
downstreamCount: selected.downstreamCount,
unresolvedParentUnknownCount: selected.unresolvedParentUnknownCount,
matches: selected.matches,
contributions: selected.contributions,
},
alphabeticalUsedAsReasoning: false,
candidates: orderedCandidates,
selected: selected
? {
nodeId: selected.nodeId,
label: selected.label,
score: selected.score,
downstreamCount: selected.downstreamCount,
unresolvedParentUnknownCount: selected.unresolvedParentUnknownCount,
matches: selected.matches,
contributions: selected.contributions,
}
: null,
competitors,
summary: {
candidateCount: candidates.length,
selectedReason: `highest_score=${selected.score}; downstream=${selected.downstreamCount}; unresolved_prerequisites=${selected.unresolvedParentUnknownCount}`,
candidateCount: orderedCandidates.length,
selectedReason: selected
? `highest_score=${selected.score}; downstream=${selected.downstreamCount}; unresolved_prerequisites=${selected.unresolvedParentUnknownCount}`
: selection.reason,
},
};
}
tests/graph/orchestrator.test.js
+81
View File
@@ -183,6 +183,87 @@ describe("lib/graph/orchestrator startCase", () => {
);
});
it("returns an ambiguous tie result instead of choosing by label order", async () => {
mockAnalyseScenario.mockResolvedValue(
makeAnalysisResult({
reconstruction: {
summary: "Revenue up while cash falls",
actors: [],
systemsOrObjects: [],
expectedStates: [],
observedStates: [
{
id: "obs-1",
label: "Revenue increased by 18%.",
description: "Revenue increased by 18%.",
confidence: "high",
},
{
id: "obs-2",
label: "Cash in the bank decreased over the same period.",
description: "Cash in the bank decreased over the same period.",
confidence: "high",
},
],
differences: [],
knownTransitions: [],
unexplainedTransitions: [],
contradictions: [
{
id: "c-1",
label:
"Apparent contradiction between profitability/revenue expansion and liquidity reduction.",
description:
"Apparent contradiction between profitability/revenue expansion and liquidity reduction.",
confidence: "medium",
},
],
importantUnknowns: [
{
id: "unk-1",
label:
"Whether revenue recognition timing differs from cash collection timing.",
description:
"Whether revenue recognition timing differs from cash collection timing.",
confidence: "high",
},
{
id: "unk-2",
label:
"Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).",
description:
"Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).",
confidence: "high",
},
],
plausibleInterpretations: [],
},
}),
);
const { startCase } = await import("@/lib/graph/orchestrator.js");
const result = await startCase({
scenario:
"Revenue increased by 18%, but cash in the bank fell over the same period.",
});
expect(result.success).toBe(true);
expect(result.situationGraph.activeUnknownNodeId).toBeNull();
expect(result.selectedQuestion).toMatchObject({
id: "q_tie_resolution",
selectionStatus: "ambiguous",
question:
"What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?",
tiedCandidateIds: expect.arrayContaining([expect.any(String)]),
});
expect(result.diagnostics.unknownSelectionExplanation).toMatchObject({
status: "ambiguous",
tieType: "complete_unresolved_tie",
selectedNodeId: null,
alphabeticalUsedAsReasoning: false,
});
});
it("returns structured failure when graph reference validation fails", async () => {
mockAnalyseScenario.mockResolvedValue(makeAnalysisResult());
const utils = await import("@/lib/graph/utils.js");
tests/graph/question-formulator.test.js
+110 -10
View File
@@ -1,6 +1,7 @@
import { describe, expect, it } from "vitest";
import {
formulateQuestion,
formulateTieResolutionQuestion,
selectInvestigationStrategy,
} from "@/lib/graph/question-formulator.js";
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
@@ -178,16 +179,24 @@ describe("formulateQuestion", () => {
});
it("the same unknown can produce different questions when paired with different strategies", () => {
const unknown = makeNode({
id: "n-same-unknown",
const thresholdUnknown = makeNode({
id: "n-threshold-unknown",
label: "Value threshold",
description: "Need to resolve the value threshold.",
kind: "unknown",
status: "unknown",
confidence: "high",
});
const definitionUnknown = makeNode({
id: "n-definition-unknown",
label: "Value term",
description: "Need to resolve what value term refers to in this context.",
kind: "unknown",
status: "unknown",
confidence: "high",
});
const decisionGraph = makeGraphFor(unknown, {
const decisionGraph = makeGraphFor(thresholdUnknown, {
centralStatement: "We are deciding whether to launch this product.",
nodes: [
makeNode({
@@ -197,35 +206,35 @@ describe("formulateQuestion", () => {
kind: "state",
status: "known",
confidence: "medium",
childIds: [unknown.id],
childIds: [thresholdUnknown.id],
value: "Deciding whether to launch the product",
}),
],
});
const definitionGraph = makeGraphFor(unknown, {
const definitionGraph = makeGraphFor(definitionUnknown, {
centralStatement:
"The team uses the term value threshold inconsistently.",
nodes: [
makeNode({
id: "n-definition",
label: "Definition disagreement",
label: "Definition disagreement about value threshold",
description:
"Need a definition of value threshold before comparing options.",
"Need a definition of value threshold because the term is used inconsistently before comparing options.",
kind: "state",
status: "known",
confidence: "medium",
childIds: [unknown.id],
childIds: [definitionUnknown.id],
}),
],
});
const decisionResult = formulateQuestion({
node: unknown,
node: thresholdUnknown,
graph: decisionGraph,
});
const definitionResult = formulateQuestion({
node: unknown,
node: definitionUnknown,
graph: definitionGraph,
});
@@ -296,4 +305,95 @@ describe("formulateQuestion", () => {
"What would resolve uncertainty regarding",
);
});
it("ambiguous contradiction produces a broad distinguishing question without accounting jargon", () => {
const unknown = makeNode({
id: "n-cause-a",
label: "Cash outflow cause",
description: "Unclear explanation for the contradiction.",
kind: "unknown",
status: "unknown",
confidence: "high",
});
const contradiction = makeNode({
id: "n-contradiction",
label: "Divergent movement between revenue and cash",
description: "Two signals moved in opposite directions.",
kind: "relationship",
status: "supported",
confidence: "medium",
});
const graph = makeGraphFor(unknown, {
centralStatement:
"Revenue increased by 18%, but cash in the bank fell over the same period.",
nodes: [contradiction],
});
const result = formulateTieResolutionQuestion({ graph });
expect(result.question).toBe(
"What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?",
);
expect(result.question.toLowerCase()).not.toMatch(
/accounts receivable|capex|debt repayments|working capital/,
);
});
it("definition is selected only for genuine definition unknowns", () => {
const unknown = makeNode({
id: "n-definition-only",
label: "Definition of success criteria",
description: "The term is used inconsistently and needs a definition.",
kind: "unknown",
status: "unknown",
confidence: "medium",
});
const result = formulateQuestion({
node: unknown,
graph: makeGraphFor(unknown),
});
expect(result.strategy).toBe("definition");
});
it("an unknown about possible causes does not become a definition question", () => {
const unknown = makeNode({
id: "n-causes",
label: "Possible causes of the divergence",
description: "Several causes may explain the divergence.",
kind: "unknown",
status: "unknown",
confidence: "medium",
});
const result = formulateQuestion({
node: unknown,
graph: makeGraphFor(unknown),
});
expect(result.strategy).toBeNull();
expect(result.question).toBe(
"What would clarify possible causes of the divergence in this situation?",
);
});
it("malformed punctuation is rejected", () => {
const unknown = makeNode({
id: "n-punct",
label: "Magnitude and nature of cash outflows (operating expenses).",
description:
"Magnitude and nature of cash outflows (operating expenses).",
kind: "unknown",
status: "unknown",
confidence: "medium",
});
const result = formulateQuestion({
node: unknown,
graph: makeGraphFor(unknown),
});
expect(result.question).not.toContain("). is true?");
expect(result.question).toBe(
"What would clarify magnitude and nature of cash outflows (operating expenses) in this situation?",
);
});
});
tests/graph/selection-influence-diagnostic.test.js
+60 -22
View File
@@ -1,7 +1,13 @@
import { describe, expect, it } from "vitest";
import { formulateQuestion } from "@/lib/graph/question-formulator.js";
import {
formulateQuestion,
formulateTieResolutionQuestion,
} from "@/lib/graph/question-formulator.js";
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
import { explainUnknownSelection } from "@/lib/graph/utils.js";
import {
explainUnknownSelection,
selectActiveUnknownCandidate,
} from "@/lib/graph/utils.js";
function buildLiveShapedGraph() {
const summary = makeNode({
@@ -184,37 +190,57 @@ function neutraliseUnknownWording(graph) {
}
describe("selection influence diagnostic", () => {
it("records ordering changes for live-shaped, structure-only, and wording-neutralised fixtures", () => {
it("records ambiguous ordering changes for live-shaped, structure-only, and wording-neutralised fixtures", () => {
const liveGraph = buildLiveShapedGraph();
const liveExplanation = explainUnknownSelection(liveGraph, []);
const liveWinner = liveGraph.nodes.find(
(node) => node.id === liveExplanation.selectedNodeId,
);
const liveQuestion = formulateQuestion({
node: liveWinner,
graph: liveGraph,
});
const liveSelection = selectActiveUnknownCandidate(liveGraph, []);
const tieQuestion = formulateTieResolutionQuestion({ graph: liveGraph });
const noLinksExplanation = explainUnknownSelection(
removeDependencyLinks(liveGraph),
[],
);
const noLinksSelection = selectActiveUnknownCandidate(
removeDependencyLinks(liveGraph),
[],
);
const neutralWordingExplanation = explainUnknownSelection(
neutraliseUnknownWording(liveGraph),
[],
);
const neutralSelection = selectActiveUnknownCandidate(
neutraliseUnknownWording(liveGraph),
[],
);
const fallbackQuestion = formulateQuestion({
node: liveGraph.nodes.find((node) => node.id === "nqdzobz"),
graph: liveGraph,
});
const diagnosticRecord = {
liveStatus: liveExplanation.status,
liveShapedCandidateOrdering: orderCandidates(liveExplanation),
liveTiedCandidateIds: liveExplanation.tiedCandidateIds,
noLinksCandidateOrdering: orderCandidates(noLinksExplanation),
noLinksStatus: noLinksExplanation.status,
neutralWordingCandidateOrdering: orderCandidates(
neutralWordingExplanation,
),
selectedExplanationContributions:
liveExplanation.selected?.contributions ?? [],
selectedInvestigationStrategy: liveQuestion.strategy,
neutralStatus: neutralWordingExplanation.status,
selectedExplanationContributions: liveExplanation.selected?.contributions,
tieQuestion: tieQuestion.question,
liveSelection,
noLinksSelection,
neutralSelection,
fallbackQuestion,
};
expect(diagnosticRecord.liveStatus).toBe("ambiguous");
expect(diagnosticRecord.liveTiedCandidateIds).toEqual([
"nqdzobz",
"niewza",
]);
expect(diagnosticRecord.liveShapedCandidateOrdering).toEqual([
{
nodeId: "nqdzobz",
@@ -233,9 +259,17 @@ describe("selection influence diagnostic", () => {
unresolvedParentUnknownCount: 0,
},
]);
expect(diagnosticRecord.liveSelection).toMatchObject({
selectedNode: null,
status: "ambiguous",
tieType: "complete_unresolved_tie",
tiedCandidateIds: ["nqdzobz", "niewza"],
});
expect(diagnosticRecord.noLinksCandidateOrdering).toEqual(
diagnosticRecord.liveShapedCandidateOrdering,
);
expect(diagnosticRecord.noLinksStatus).toBe("ambiguous");
expect(diagnosticRecord.noLinksSelection.status).toBe("ambiguous");
expect(diagnosticRecord.neutralWordingCandidateOrdering).toEqual([
{
nodeId: "niewza",
@@ -252,14 +286,18 @@ describe("selection influence diagnostic", () => {
unresolvedParentUnknownCount: 0,
},
]);
expect(diagnosticRecord.selectedExplanationContributions).toEqual([
{
rule: "downstream_dependencies",
value: 0,
weight: 4,
delta: 0,
},
]);
expect(diagnosticRecord.selectedInvestigationStrategy).toBe("definition");
expect(diagnosticRecord.neutralStatus).toBe("ambiguous");
expect(diagnosticRecord.neutralSelection.status).toBe("ambiguous");
expect(diagnosticRecord.selectedExplanationContributions).toBeUndefined();
expect(diagnosticRecord.tieQuestion).toBe(
"What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?",
);
expect(diagnosticRecord.tieQuestion.toLowerCase()).not.toMatch(
/accounts receivable|capex|debt repayments|working capital/,
);
expect(diagnosticRecord.fallbackQuestion.strategy).toBeNull();
expect(diagnosticRecord.fallbackQuestion.question).toBe(
"What would clarify magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts) in this situation?",
);
});
});
tests/graph/utils.test.js
+67
View File
@@ -456,6 +456,7 @@ describe("selectActiveUnknownCandidate", () => {
const result = selectActiveUnknownCandidate(graph, []);
expect(result.nodeId).toBe("unknown-a"); // Has more dependents (score 2 vs 0)
expect(result.status).toBe("selected");
});
it("returns one candidate (not array)", () => {
@@ -619,6 +620,72 @@ describe("selectActiveUnknownCandidate", () => {
const childScore = scoreUnknownCandidate(graph, childUnknown, []);
expect(parentScore.score).toBeGreaterThan(childScore.score);
});
it("returns ambiguous for a complete unresolved tie instead of label-based winner", () => {
const unknownA = makeNode({
id: "tie-a",
label: "Magnitude and nature of cash outflows",
description: "Magnitude and nature of cash outflows.",
kind: "unknown",
status: "unknown",
confidence: "high",
});
const unknownB = makeNode({
id: "tie-b",
label:
"Whether revenue recognition timing differs from cash collection timing",
description:
"Whether revenue recognition timing differs from cash collection timing.",
kind: "unknown",
status: "unknown",
confidence: "high",
});
const graph = makeGraph({
centralStatement:
"Revenue increased by 18%, but cash in the bank fell over the same period.",
nodes: [unknownA, unknownB],
edges: [],
activeUnknownNodeId: null,
resolvedNodeIds: [],
currentSummary: "Tie case",
});
const result = selectActiveUnknownCandidate(graph, []);
expect(result).toMatchObject({
selectedNode: null,
status: "ambiguous",
tieType: "complete_unresolved_tie",
tiedCandidateIds: ["tie-a", "tie-b"],
});
expect(result.nodeId).toBeUndefined();
});
it("alphabetical renaming does not resolve a complete tie", () => {
const unknownA = makeNode({
id: "tie-a",
label: "Unknown B",
description: "Unknown factor one.",
kind: "unknown",
});
const unknownB = makeNode({
id: "tie-b",
label: "Unknown A",
description: "Unknown factor two.",
kind: "unknown",
});
const graph = makeGraph({
centralStatement: "Two conflicting signals remain unresolved.",
nodes: [unknownA, unknownB],
edges: [],
activeUnknownNodeId: null,
resolvedNodeIds: [],
currentSummary: "Tie case",
});
const result = selectActiveUnknownCandidate(graph, []);
expect(result.status).toBe("ambiguous");
expect(result.tiedCandidateIds.sort()).toEqual(["tie-a", "tie-b"]);
});
});
describe("applyGraphUpdate", () => {