fix: handle unjustified unknown selection ties

This commit is contained in:
2026-08-02 16:09:00 +01:00
parent a1f6d0c2b9
commit 51ce356218
8 changed files with 731 additions and 145 deletions
+35 -17
View File
@@ -1,5 +1,8 @@
import { describeGraph } from "./builder.js"; import { describeGraph } from "./builder.js";
import { formulateQuestion } from "./question-formulator.js"; import {
formulateQuestion,
formulateTieResolutionQuestion,
} from "./question-formulator.js";
import { graphUpdateSchema, situationGraphSchema } from "./schema.js"; import { graphUpdateSchema, situationGraphSchema } from "./schema.js";
import { import {
applyGraphUpdate, applyGraphUpdate,
@@ -623,18 +626,25 @@ export function applyValidatedProposal({ situationGraph, proposal }) {
updatedSituationGraph.resolvedNodeIds, updatedSituationGraph.resolvedNodeIds,
); );
if (deterministicSelection?.nodeId) { if (
deterministicSelection?.status === "selected" &&
deterministicSelection?.nodeId
) {
newActiveUnknownNodeId = deterministicSelection.nodeId; newActiveUnknownNodeId = deterministicSelection.nodeId;
} else if (deterministicSelection?.status === "ambiguous") {
newActiveUnknownNodeId = null;
} }
updatedSituationGraph.activeUnknownNodeId = newActiveUnknownNodeId; updatedSituationGraph.activeUnknownNodeId = newActiveUnknownNodeId;
updatedSituationGraph.currentSummary = describeGraph(updatedSituationGraph); updatedSituationGraph.currentSummary = describeGraph(updatedSituationGraph);
const selectedNode = deterministicSelection?.nodeId const selectedNode =
? updatedSituationGraph.nodes.find( deterministicSelection?.status === "selected" &&
(node) => node.id === deterministicSelection.nodeId, deterministicSelection?.nodeId
) ? updatedSituationGraph.nodes.find(
: null; (node) => node.id === deterministicSelection.nodeId,
)
: null;
const formulatedQuestion = selectedNode const formulatedQuestion = selectedNode
? formulateQuestion({ ? formulateQuestion({
node: selectedNode, node: selectedNode,
@@ -645,20 +655,28 @@ export function applyValidatedProposal({ situationGraph, proposal }) {
.filter( .filter(
(value) => typeof value === "string" && value.trim().length > 0, (value) => typeof value === "string" && value.trim().length > 0,
), ),
selectionState: deterministicSelection,
}, },
}) })
: null; : null;
const finalSelectedQuestion = deterministicSelection const finalSelectedQuestion =
? { deterministicSelection?.status === "ambiguous"
nodeId: deterministicSelection.nodeId, ? {
question: nodeId: null,
formulatedQuestion?.question || deterministicSelection.question, tiedCandidateIds: deterministicSelection.tiedCandidateIds,
reason: formulatedQuestion?.reason || deterministicSelection.reason, ...formulateTieResolutionQuestion({ graph: updatedSituationGraph }),
strategy: formulatedQuestion?.strategy, }
investigationStrategy: formulatedQuestion?.investigationStrategy, : deterministicSelection?.status === "selected"
} ? {
: null; nodeId: deterministicSelection.nodeId,
question:
formulatedQuestion?.question || deterministicSelection.question,
reason: formulatedQuestion?.reason || deterministicSelection.reason,
strategy: formulatedQuestion?.strategy,
investigationStrategy: formulatedQuestion?.investigationStrategy,
}
: null;
const resultGraphValidation = situationGraphSchema.safeParse( const resultGraphValidation = situationGraphSchema.safeParse(
updatedSituationGraph, updatedSituationGraph,
+46 -11
View File
@@ -15,6 +15,7 @@ import {
import { buildInitialGraph, describeGraph } from "./builder.js"; import { buildInitialGraph, describeGraph } from "./builder.js";
import { applyValidatedProposal } from "./apply-proposal.js"; import { applyValidatedProposal } from "./apply-proposal.js";
import { buildGraphUpdatePrompt } from "./prompt-builder.js"; import { buildGraphUpdatePrompt } from "./prompt-builder.js";
import { formulateTieResolutionQuestion } from "./question-formulator.js";
import { parseGraphUpdateProposal } from "./update-proposal.js"; import { parseGraphUpdateProposal } from "./update-proposal.js";
import { import {
explainUnknownSelection, explainUnknownSelection,
@@ -53,6 +54,26 @@ function buildDiagnostics({
}; };
} }
function buildUnknownSelectionDiagnostics(
graph,
resolvedNodeIds = [],
selectedQuestion = null,
) {
const explanation = explainUnknownSelection(graph, resolvedNodeIds);
if (explanation.status === "ambiguous") {
return {
...explanation,
tieResolutionQuestion:
selectedQuestion?.selectionStatus === "ambiguous"
? selectedQuestion.question
: formulateTieResolutionQuestion({ graph }).question,
alphabeticalUsedAsReasoning: false,
};
}
return explanation;
}
function buildUpdateDiagnostics({ function buildUpdateDiagnostics({
promptVersion, promptVersion,
modelName, modelName,
@@ -119,14 +140,17 @@ export async function startCase(body) {
}); });
const currentSummary = describeGraph(initialGraph); const currentSummary = describeGraph(initialGraph);
const deterministicSelection = selectActiveUnknownCandidate(
{
...initialGraph,
resolvedNodeIds: [],
},
[],
);
const activeUnknownNodeId = const activeUnknownNodeId =
selectActiveUnknownCandidate( deterministicSelection?.status === "selected"
{ ? deterministicSelection.nodeId
...initialGraph, : null;
resolvedNodeIds: [],
},
[],
)?.nodeId ?? null;
const situationGraph = makeGraph({ const situationGraph = makeGraph({
centralStatement: scenario, centralStatement: scenario,
@@ -140,9 +164,18 @@ export async function startCase(body) {
situationGraphSchema.parse(situationGraph); situationGraphSchema.parse(situationGraph);
const graphReferenceValidation = validateGraphReferences(situationGraph); const graphReferenceValidation = validateGraphReferences(situationGraph);
const unknownSelectionExplanation = explainUnknownSelection( const selectedQuestion =
deterministicSelection?.status === "ambiguous"
? {
id: "q_tie_resolution",
...formulateTieResolutionQuestion({ graph: situationGraph }),
tiedCandidateIds: deterministicSelection.tiedCandidateIds,
}
: (analysis.nextQuestion ?? null);
const unknownSelectionExplanation = buildUnknownSelectionDiagnostics(
situationGraph, situationGraph,
[], [],
selectedQuestion,
); );
if (!graphReferenceValidation.valid) { if (!graphReferenceValidation.valid) {
return { return {
@@ -162,7 +195,7 @@ export async function startCase(body) {
return { return {
success: true, success: true,
situationGraph, situationGraph,
selectedQuestion: analysis.nextQuestion ?? null, selectedQuestion,
diagnostics: buildDiagnostics({ diagnostics: buildDiagnostics({
analysis, analysis,
graph: situationGraph, graph: situationGraph,
@@ -339,9 +372,10 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
graph: applicationResult.updatedSituationGraph, graph: applicationResult.updatedSituationGraph,
graphReferenceValidation: applicationResult.graphReferenceValidation, graphReferenceValidation: applicationResult.graphReferenceValidation,
selectedQuestion: applicationResult.selectedQuestion, selectedQuestion: applicationResult.selectedQuestion,
unknownSelectionExplanation: explainUnknownSelection( unknownSelectionExplanation: buildUnknownSelectionDiagnostics(
applicationResult.updatedSituationGraph, applicationResult.updatedSituationGraph,
applicationResult.updatedSituationGraph.resolvedNodeIds || [], applicationResult.updatedSituationGraph.resolvedNodeIds || [],
applicationResult.selectedQuestion,
), ),
}), }),
}; };
@@ -359,9 +393,10 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
graph: situationGraph, graph: situationGraph,
graphReferenceValidation, graphReferenceValidation,
selectedQuestion: null, selectedQuestion: null,
unknownSelectionExplanation: explainUnknownSelection( unknownSelectionExplanation: buildUnknownSelectionDiagnostics(
situationGraph, situationGraph,
situationGraph.resolvedNodeIds || [], situationGraph.resolvedNodeIds || [],
null,
), ),
}), }),
}; };
+132 -23
View File
@@ -11,6 +11,13 @@ function sentenceCase(value) {
return trimmed.charAt(0).toLowerCase() + trimmed.slice(1); return trimmed.charAt(0).toLowerCase() + trimmed.slice(1);
} }
function stripTrailingPunctuation(value) {
return String(value || "")
.trim()
.replace(/[.?!:;]+$/g, "")
.trim();
}
function buildNodeMap(graph) { function buildNodeMap(graph) {
return new Map((graph?.nodes || []).map((node) => [node.id, node])); return new Map((graph?.nodes || []).map((node) => [node.id, node]));
} }
@@ -52,8 +59,8 @@ function collectResolvedContextValues(graph) {
function extractMeaning(node) { function extractMeaning(node) {
const raw = `${node?.label || ""} ${node?.description || ""}`.trim(); const raw = `${node?.label || ""} ${node?.description || ""}`.trim();
let meaning = String( let meaning = stripTrailingPunctuation(
node?.label || node?.description || "this uncertainty", String(node?.label || node?.description || "this uncertainty"),
).trim(); ).trim();
const lowered = normaliseText(raw); const lowered = normaliseText(raw);
@@ -79,6 +86,84 @@ function extractMeaning(node) {
return sentenceCase(meaning); return sentenceCase(meaning);
} }
function isDefinitionLikeUnknown(nodeText, text) {
return (
/\b(define|definition|meaning|term|terminology)\b/.test(nodeText) ||
(/\bdefinition\b/.test(text) && /\bdisagreement\b/.test(text)) ||
(/\b(define|definition|meaning|term|terminology)\b/.test(text) &&
/\b(unclear|ambiguous|inconsistent|undefined|used inconsistently)\b/.test(
text,
))
);
}
function isClaimLikeUnknown(node, text) {
return (
node?.kind === "reported_claim" ||
node?.kind === "conclusion" ||
/\b(claim|assertion|true|false|correct|incorrect|happened|happening)\b/.test(
text,
) ||
/^whether\b/i.test(String(node?.label || "").trim())
);
}
function sanitizeQuestionText(question) {
return String(question || "")
.replace(/\)\.\s+/g, ") ")
.replace(/\s+/g, " ")
.trim();
}
function buildNeutralClarificationQuestion(meaning) {
return `What would clarify ${stripTrailingPunctuation(meaning)} in this situation?`;
}
function buildEvidenceFallbackQuestion(meaning) {
return `What evidence would confirm or rule out ${stripTrailingPunctuation(meaning)}?`;
}
function detectContradictionContext(graph) {
const central = stripTrailingPunctuation(
graph?.centralStatement || "this situation",
);
const contradictionNode = (graph?.nodes || []).find((node) => {
const text = normaliseText(`${node.label} ${node.description}`);
return (
node.kind === "relationship" &&
/\b(contradiction|conflict|inconsistent|mismatch|divergent|opposing)\b/.test(
text,
)
);
});
return {
centralStatement: central,
contradictionLabel: stripTrailingPunctuation(
contradictionNode?.label || "",
),
};
}
export function formulateTieResolutionQuestion({ graph }) {
const { centralStatement, contradictionLabel } =
detectContradictionContext(graph);
const focus =
centralStatement || contradictionLabel || "these conflicting signals";
const question = sanitizeQuestionText(
`What changed during the period that could explain why ${focus}?`,
);
return {
question,
reason:
"Formulated to distinguish between tied unresolved explanations without prematurely choosing one branch.",
strategy: null,
investigationStrategy: null,
selectionStatus: "ambiguous",
};
}
function extractActionPhrase(texts) { function extractActionPhrase(texts) {
for (const text of texts) { for (const text of texts) {
const value = String(text || "").trim(); const value = String(text || "").trim();
@@ -224,8 +309,7 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) {
}); });
} }
const hasDefinitionLanguage = const hasDefinitionLanguage = isDefinitionLikeUnknown(nodeText, text);
/\b(define|definition|meaning|term|terminology)\b/.test(text);
const hasPrimaryDefinitionLanguage = const hasPrimaryDefinitionLanguage =
/\b(define|definition|meaning|term|terminology)\b/.test(nodeText); /\b(define|definition|meaning|term|terminology)\b/.test(nodeText);
const hasCriteriaLanguage = const hasCriteriaLanguage =
@@ -243,9 +327,7 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) {
); );
const hasEvidenceLanguage = const hasEvidenceLanguage =
/\b(evidence|proof|validate|validation|signal|demand)\b/.test(text) || /\b(evidence|proof|validate|validation|signal|demand)\b/.test(text) ||
node?.kind === "reported_claim" || isClaimLikeUnknown(node, text);
node?.kind === "conclusion" ||
/\b(claim|assertion|true|false)\b/.test(text);
const hasContradictionLanguage = const hasContradictionLanguage =
/\b(contradiction|contradict|conflict|inconsistent|inconsistency|disagree|mismatch)\b/.test( /\b(contradiction|contradict|conflict|inconsistent|inconsistency|disagree|mismatch)\b/.test(
`${text} ${relatedText}`, `${text} ${relatedText}`,
@@ -323,16 +405,7 @@ export function selectInvestigationStrategy({ node, graph, context = {} }) {
}); });
} }
return buildInvestigationStrategy({ return null;
key: "definition",
reason:
"Selected as the deterministic fallback because clarifying the exact meaning of the unknown is the narrowest first step.",
node,
graph,
relatedNodes,
meaning,
actionPhrase,
});
} }
function buildQuestionFromStrategy(strategy) { function buildQuestionFromStrategy(strategy) {
@@ -344,7 +417,7 @@ function buildQuestionFromStrategy(strategy) {
case "definition": case "definition":
return `What does ${strategy.meaning} mean in this situation?`; return `What does ${strategy.meaning} mean in this situation?`;
case "evidence_gathering": case "evidence_gathering":
return `What evidence would show whether ${strategy.meaning} is true?`; return `What evidence would clarify ${stripTrailingPunctuation(strategy.meaning)}?`;
case "baseline_reconstruction": case "baseline_reconstruction":
return `What was the comparable state before ${strategy.meaning}?`; return `What was the comparable state before ${strategy.meaning}?`;
case "contradiction_resolution": case "contradiction_resolution":
@@ -382,6 +455,9 @@ function validateFormulatedQuestion(question, meaning) {
if (/^how should uncertainty regarding\b/i.test(trimmed)) return false; if (/^how should uncertainty regarding\b/i.test(trimmed)) return false;
if (/^what would resolve uncertainty regarding\b/i.test(trimmed)) if (/^what would resolve uncertainty regarding\b/i.test(trimmed))
return false; return false;
if (/\)\.\s+[A-Z]/.test(trimmed)) return false;
if (/\bis true\?$/i.test(trimmed) && !/^whether\b/i.test(meaning))
return false;
if ( if (
/\bprice|pricing|price point\b/i.test(trimmed) && /\bprice|pricing|price point\b/i.test(trimmed) &&
!/\bprice\b/i.test(meaning) !/\bprice\b/i.test(meaning)
@@ -399,22 +475,55 @@ function validateFormulatedQuestion(question, meaning) {
} }
export function formulateQuestion({ node, graph, context = {} }) { export function formulateQuestion({ node, graph, context = {} }) {
if (context.selectionState?.status === "ambiguous") {
return formulateTieResolutionQuestion({ graph });
}
const investigationStrategy = selectInvestigationStrategy({ const investigationStrategy = selectInvestigationStrategy({
node, node,
graph, graph,
context, context,
}); });
let question = buildQuestionFromStrategy(investigationStrategy); let question = investigationStrategy
? buildQuestionFromStrategy(investigationStrategy)
: buildNeutralClarificationQuestion(extractMeaning(node));
if (!validateFormulatedQuestion(question, investigationStrategy.meaning)) { question = sanitizeQuestionText(question);
question = `What evidence would resolve whether ${investigationStrategy.meaning} is true?`;
const fallbackMeaning = extractMeaning(node);
if (
!validateFormulatedQuestion(
question,
investigationStrategy?.meaning || fallbackMeaning,
)
) {
question = sanitizeQuestionText(
investigationStrategy &&
isClaimLikeUnknown(
node,
normaliseText(
collectRelatedNodes(node, graph)
.map(
(relatedNode) =>
`${relatedNode.label} ${relatedNode.description}`,
)
.concat([node?.label, node?.description])
.filter(Boolean)
.join(" "),
),
)
? buildEvidenceFallbackQuestion(fallbackMeaning)
: buildNeutralClarificationQuestion(fallbackMeaning),
);
} }
return { return {
question, question,
reason: `Formulated from graph context using the ${investigationStrategy.key} investigation strategy.`, reason: investigationStrategy
strategy: investigationStrategy.key, ? `Formulated from graph context using the ${investigationStrategy.key} investigation strategy.`
: "Formulated as a neutral clarification question because no narrower investigation strategy clearly applied.",
strategy: investigationStrategy?.key ?? null,
investigationStrategy, investigationStrategy,
}; };
} }
+200 -62
View File
@@ -216,6 +216,119 @@ function buildScoreContributions(
return contributions; return contributions;
} }
function getMeaningfulSemanticContributions(contributions = []) {
return contributions
.filter(
(contribution) =>
contribution.rule !== "downstream_dependencies" &&
contribution.rule !== "unresolved_prerequisite_penalty" &&
contribution.delta !== 0,
)
.map((contribution) => ({
rule: contribution.rule,
delta: contribution.delta,
}));
}
function buildCandidateDisplayOrder(candidates) {
return [...candidates].sort((a, b) => {
if (b.score !== a.score) return b.score - a.score;
if (b.downstreamCount !== a.downstreamCount) {
return b.downstreamCount - a.downstreamCount;
}
if (a.unresolvedParentUnknownCount !== b.unresolvedParentUnknownCount) {
return a.unresolvedParentUnknownCount - b.unresolvedParentUnknownCount;
}
return a.label.localeCompare(b.label);
});
}
function semanticSignature(candidate) {
return JSON.stringify(
getMeaningfulSemanticContributions(candidate.contributions),
);
}
function classifyCandidateOrdering(candidates) {
const displayOrder = buildCandidateDisplayOrder(candidates);
const best = displayOrder[0] ?? null;
if (!best) {
return {
displayOrder,
best: null,
leadingCandidates: [],
status: "no_candidates",
tieType: "none",
usedAlphabeticalOrdering: false,
reason: "No unresolved unknown candidates remain.",
};
}
const topScoreCandidates = displayOrder.filter(
(candidate) => candidate.score === best.score,
);
if (topScoreCandidates.length === 1) {
return {
displayOrder,
best,
leadingCandidates: [best],
status: "selected",
tieType: "none",
usedAlphabeticalOrdering: false,
reason: `Clear winner by total score (${best.score}).`,
};
}
const topStructuralCandidates = topScoreCandidates.filter(
(candidate) =>
candidate.downstreamCount === best.downstreamCount &&
candidate.unresolvedParentUnknownCount ===
best.unresolvedParentUnknownCount,
);
if (topStructuralCandidates.length === 1) {
return {
displayOrder,
best,
leadingCandidates: [best],
status: "selected",
tieType: "structural_tie",
usedAlphabeticalOrdering: false,
reason:
"Score tie was resolved by downstream dependency count or prerequisite ordering.",
};
}
const topSemanticSignature = semanticSignature(best);
const semanticPeers = topStructuralCandidates.filter(
(candidate) => semanticSignature(candidate) === topSemanticSignature,
);
if (semanticPeers.length !== topStructuralCandidates.length) {
return {
displayOrder,
best: null,
leadingCandidates: topStructuralCandidates,
status: "ambiguous",
tieType: "semantic_tie",
usedAlphabeticalOrdering: false,
reason:
"Leading candidates remain tied after score and structural checks, but differ in semantic contribution patterns.",
};
}
return {
displayOrder,
best: null,
leadingCandidates: topStructuralCandidates,
status: "ambiguous",
tieType: "complete_unresolved_tie",
usedAlphabeticalOrdering: false,
reason: "No justified distinction between leading unknowns.",
};
}
export function scoreUnknownCandidate(graph, node, resolvedNodeIds = []) { export function scoreUnknownCandidate(graph, node, resolvedNodeIds = []) {
const text = collectNodeText(node); const text = collectNodeText(node);
const matches = classifyUnknownPriority(text); const matches = classifyUnknownPriority(text);
@@ -490,21 +603,36 @@ export function selectActiveUnknownCandidate(graph, resolvedNodeIds) {
...scoreUnknownCandidate(graph, node, resolvedNodeIds), ...scoreUnknownCandidate(graph, node, resolvedNodeIds),
})); }));
scoredCandidates.sort((a, b) => { const selection = classifyCandidateOrdering(
if (b.score !== a.score) return b.score - a.score; scoredCandidates.map(({ node, ...candidate }) => ({
if (b.downstreamCount !== a.downstreamCount) { ...candidate,
return b.downstreamCount - a.downstreamCount; node,
} })),
if (a.unresolvedParentUnknownCount !== b.unresolvedParentUnknownCount) { );
return a.unresolvedParentUnknownCount - b.unresolvedParentUnknownCount;
}
return a.node.label.localeCompare(b.node.label);
});
const best = scoredCandidates[0]; if (selection.status === "ambiguous") {
return {
selectedNode: null,
status: "ambiguous",
tieType: selection.tieType,
tiedCandidateIds: selection.leadingCandidates.map(
(candidate) => candidate.nodeId,
),
displayOrder: selection.displayOrder.map((candidate) => candidate.nodeId),
reason: selection.reason,
};
}
const best = selection.best;
if (!best) return null; if (!best) return null;
return { return {
selectedNode: {
nodeId: best.node.id,
label: best.node.label,
},
status: "selected",
tieType: selection.tieType,
nodeId: best.node.id, nodeId: best.node.id,
label: best.node.label, label: best.node.label,
score: best.score, score: best.score,
@@ -522,7 +650,10 @@ export function explainUnknownSelection(graph, resolvedNodeIds = []) {
return { return {
selectedNodeId: null, selectedNodeId: null,
selectedNodeLabel: null, selectedNodeLabel: null,
status: "no_candidates",
tieType: "none",
resolvedNodeIds: [...resolvedNodeIds], resolvedNodeIds: [...resolvedNodeIds],
tiedCandidateIds: [],
candidates: [], candidates: [],
competitors: [], competitors: [],
tieBreakOrder: [ tieBreakOrder: [
@@ -543,68 +674,75 @@ export function explainUnknownSelection(graph, resolvedNodeIds = []) {
...scoreUnknownCandidate(graph, node, resolvedNodeIds), ...scoreUnknownCandidate(graph, node, resolvedNodeIds),
})); }));
candidates.sort((a, b) => { const selection = classifyCandidateOrdering(candidates);
if (b.score !== a.score) return b.score - a.score; const orderedCandidates = selection.displayOrder;
if (b.downstreamCount !== a.downstreamCount) { const selected = selection.best;
return b.downstreamCount - a.downstreamCount; const competitors = orderedCandidates
} .filter((candidate) => candidate.nodeId !== selected?.nodeId)
if (a.unresolvedParentUnknownCount !== b.unresolvedParentUnknownCount) { .map((candidate) => ({
return a.unresolvedParentUnknownCount - b.unresolvedParentUnknownCount; nodeId: candidate.nodeId,
} label: candidate.label,
return a.label.localeCompare(b.label); score: candidate.score,
}); downstreamCount: candidate.downstreamCount,
unresolvedParentUnknownCount: candidate.unresolvedParentUnknownCount,
const selected = candidates[0]; matches: candidate.matches,
const competitors = candidates.slice(1).map((candidate) => ({ contributions: candidate.contributions,
nodeId: candidate.nodeId, outrankedBy: {
label: candidate.label, scoreDelta: (selected?.score ?? candidate.score) - candidate.score,
score: candidate.score, downstreamDelta:
downstreamCount: candidate.downstreamCount, (selected?.downstreamCount ?? candidate.downstreamCount) -
unresolvedParentUnknownCount: candidate.unresolvedParentUnknownCount, candidate.downstreamCount,
matches: candidate.matches, unresolvedPrerequisiteDelta:
contributions: candidate.contributions, candidate.unresolvedParentUnknownCount -
outrankedBy: { (selected?.unresolvedParentUnknownCount ??
scoreDelta: selected.score - candidate.score, candidate.unresolvedParentUnknownCount),
downstreamDelta: selected.downstreamCount - candidate.downstreamCount, labelOrderWinner:
unresolvedPrerequisiteDelta: selected &&
candidate.unresolvedParentUnknownCount - selected.score === candidate.score &&
selected.unresolvedParentUnknownCount, selected.downstreamCount === candidate.downstreamCount &&
labelOrderWinner: selected.unresolvedParentUnknownCount ===
selected.score === candidate.score && candidate.unresolvedParentUnknownCount
selected.downstreamCount === candidate.downstreamCount && ? selected.label.localeCompare(candidate.label) <= 0
selected.unresolvedParentUnknownCount === ? selected.label
candidate.unresolvedParentUnknownCount : candidate.label
? selected.label.localeCompare(candidate.label) <= 0 : null,
? selected.label },
: candidate.label }));
: null,
},
}));
return { return {
selectedNodeId: selected.nodeId, selectedNodeId: selected?.nodeId ?? null,
selectedNodeLabel: selected.label, selectedNodeLabel: selected?.label ?? null,
status: selection.status,
tieType: selection.tieType,
resolvedNodeIds: [...resolvedNodeIds], resolvedNodeIds: [...resolvedNodeIds],
tiedCandidateIds: selection.leadingCandidates.map(
(candidate) => candidate.nodeId,
),
tieBreakOrder: [ tieBreakOrder: [
"score_desc", "score_desc",
"downstreamCount_desc", "downstreamCount_desc",
"unresolvedParentUnknownCount_asc", "unresolvedParentUnknownCount_asc",
"label_asc", "label_asc",
], ],
candidates, alphabeticalUsedAsReasoning: false,
selected: { candidates: orderedCandidates,
nodeId: selected.nodeId, selected: selected
label: selected.label, ? {
score: selected.score, nodeId: selected.nodeId,
downstreamCount: selected.downstreamCount, label: selected.label,
unresolvedParentUnknownCount: selected.unresolvedParentUnknownCount, score: selected.score,
matches: selected.matches, downstreamCount: selected.downstreamCount,
contributions: selected.contributions, unresolvedParentUnknownCount: selected.unresolvedParentUnknownCount,
}, matches: selected.matches,
contributions: selected.contributions,
}
: null,
competitors, competitors,
summary: { summary: {
candidateCount: candidates.length, candidateCount: orderedCandidates.length,
selectedReason: `highest_score=${selected.score}; downstream=${selected.downstreamCount}; unresolved_prerequisites=${selected.unresolvedParentUnknownCount}`, selectedReason: selected
? `highest_score=${selected.score}; downstream=${selected.downstreamCount}; unresolved_prerequisites=${selected.unresolvedParentUnknownCount}`
: selection.reason,
}, },
}; };
} }
+81
View File
@@ -183,6 +183,87 @@ describe("lib/graph/orchestrator startCase", () => {
); );
}); });
it("returns an ambiguous tie result instead of choosing by label order", async () => {
mockAnalyseScenario.mockResolvedValue(
makeAnalysisResult({
reconstruction: {
summary: "Revenue up while cash falls",
actors: [],
systemsOrObjects: [],
expectedStates: [],
observedStates: [
{
id: "obs-1",
label: "Revenue increased by 18%.",
description: "Revenue increased by 18%.",
confidence: "high",
},
{
id: "obs-2",
label: "Cash in the bank decreased over the same period.",
description: "Cash in the bank decreased over the same period.",
confidence: "high",
},
],
differences: [],
knownTransitions: [],
unexplainedTransitions: [],
contradictions: [
{
id: "c-1",
label:
"Apparent contradiction between profitability/revenue expansion and liquidity reduction.",
description:
"Apparent contradiction between profitability/revenue expansion and liquidity reduction.",
confidence: "medium",
},
],
importantUnknowns: [
{
id: "unk-1",
label:
"Whether revenue recognition timing differs from cash collection timing.",
description:
"Whether revenue recognition timing differs from cash collection timing.",
confidence: "high",
},
{
id: "unk-2",
label:
"Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).",
description:
"Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).",
confidence: "high",
},
],
plausibleInterpretations: [],
},
}),
);
const { startCase } = await import("@/lib/graph/orchestrator.js");
const result = await startCase({
scenario:
"Revenue increased by 18%, but cash in the bank fell over the same period.",
});
expect(result.success).toBe(true);
expect(result.situationGraph.activeUnknownNodeId).toBeNull();
expect(result.selectedQuestion).toMatchObject({
id: "q_tie_resolution",
selectionStatus: "ambiguous",
question:
"What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?",
tiedCandidateIds: expect.arrayContaining([expect.any(String)]),
});
expect(result.diagnostics.unknownSelectionExplanation).toMatchObject({
status: "ambiguous",
tieType: "complete_unresolved_tie",
selectedNodeId: null,
alphabeticalUsedAsReasoning: false,
});
});
it("returns structured failure when graph reference validation fails", async () => { it("returns structured failure when graph reference validation fails", async () => {
mockAnalyseScenario.mockResolvedValue(makeAnalysisResult()); mockAnalyseScenario.mockResolvedValue(makeAnalysisResult());
const utils = await import("@/lib/graph/utils.js"); const utils = await import("@/lib/graph/utils.js");
+110 -10
View File
@@ -1,6 +1,7 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import { import {
formulateQuestion, formulateQuestion,
formulateTieResolutionQuestion,
selectInvestigationStrategy, selectInvestigationStrategy,
} from "@/lib/graph/question-formulator.js"; } from "@/lib/graph/question-formulator.js";
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js"; import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
@@ -178,16 +179,24 @@ describe("formulateQuestion", () => {
}); });
it("the same unknown can produce different questions when paired with different strategies", () => { it("the same unknown can produce different questions when paired with different strategies", () => {
const unknown = makeNode({ const thresholdUnknown = makeNode({
id: "n-same-unknown", id: "n-threshold-unknown",
label: "Value threshold", label: "Value threshold",
description: "Need to resolve the value threshold.", description: "Need to resolve the value threshold.",
kind: "unknown", kind: "unknown",
status: "unknown", status: "unknown",
confidence: "high", confidence: "high",
}); });
const definitionUnknown = makeNode({
id: "n-definition-unknown",
label: "Value term",
description: "Need to resolve what value term refers to in this context.",
kind: "unknown",
status: "unknown",
confidence: "high",
});
const decisionGraph = makeGraphFor(unknown, { const decisionGraph = makeGraphFor(thresholdUnknown, {
centralStatement: "We are deciding whether to launch this product.", centralStatement: "We are deciding whether to launch this product.",
nodes: [ nodes: [
makeNode({ makeNode({
@@ -197,35 +206,35 @@ describe("formulateQuestion", () => {
kind: "state", kind: "state",
status: "known", status: "known",
confidence: "medium", confidence: "medium",
childIds: [unknown.id], childIds: [thresholdUnknown.id],
value: "Deciding whether to launch the product", value: "Deciding whether to launch the product",
}), }),
], ],
}); });
const definitionGraph = makeGraphFor(unknown, { const definitionGraph = makeGraphFor(definitionUnknown, {
centralStatement: centralStatement:
"The team uses the term value threshold inconsistently.", "The team uses the term value threshold inconsistently.",
nodes: [ nodes: [
makeNode({ makeNode({
id: "n-definition", id: "n-definition",
label: "Definition disagreement", label: "Definition disagreement about value threshold",
description: description:
"Need a definition of value threshold before comparing options.", "Need a definition of value threshold because the term is used inconsistently before comparing options.",
kind: "state", kind: "state",
status: "known", status: "known",
confidence: "medium", confidence: "medium",
childIds: [unknown.id], childIds: [definitionUnknown.id],
}), }),
], ],
}); });
const decisionResult = formulateQuestion({ const decisionResult = formulateQuestion({
node: unknown, node: thresholdUnknown,
graph: decisionGraph, graph: decisionGraph,
}); });
const definitionResult = formulateQuestion({ const definitionResult = formulateQuestion({
node: unknown, node: definitionUnknown,
graph: definitionGraph, graph: definitionGraph,
}); });
@@ -296,4 +305,95 @@ describe("formulateQuestion", () => {
"What would resolve uncertainty regarding", "What would resolve uncertainty regarding",
); );
}); });
it("ambiguous contradiction produces a broad distinguishing question without accounting jargon", () => {
const unknown = makeNode({
id: "n-cause-a",
label: "Cash outflow cause",
description: "Unclear explanation for the contradiction.",
kind: "unknown",
status: "unknown",
confidence: "high",
});
const contradiction = makeNode({
id: "n-contradiction",
label: "Divergent movement between revenue and cash",
description: "Two signals moved in opposite directions.",
kind: "relationship",
status: "supported",
confidence: "medium",
});
const graph = makeGraphFor(unknown, {
centralStatement:
"Revenue increased by 18%, but cash in the bank fell over the same period.",
nodes: [contradiction],
});
const result = formulateTieResolutionQuestion({ graph });
expect(result.question).toBe(
"What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?",
);
expect(result.question.toLowerCase()).not.toMatch(
/accounts receivable|capex|debt repayments|working capital/,
);
});
it("definition is selected only for genuine definition unknowns", () => {
const unknown = makeNode({
id: "n-definition-only",
label: "Definition of success criteria",
description: "The term is used inconsistently and needs a definition.",
kind: "unknown",
status: "unknown",
confidence: "medium",
});
const result = formulateQuestion({
node: unknown,
graph: makeGraphFor(unknown),
});
expect(result.strategy).toBe("definition");
});
it("an unknown about possible causes does not become a definition question", () => {
const unknown = makeNode({
id: "n-causes",
label: "Possible causes of the divergence",
description: "Several causes may explain the divergence.",
kind: "unknown",
status: "unknown",
confidence: "medium",
});
const result = formulateQuestion({
node: unknown,
graph: makeGraphFor(unknown),
});
expect(result.strategy).toBeNull();
expect(result.question).toBe(
"What would clarify possible causes of the divergence in this situation?",
);
});
it("malformed punctuation is rejected", () => {
const unknown = makeNode({
id: "n-punct",
label: "Magnitude and nature of cash outflows (operating expenses).",
description:
"Magnitude and nature of cash outflows (operating expenses).",
kind: "unknown",
status: "unknown",
confidence: "medium",
});
const result = formulateQuestion({
node: unknown,
graph: makeGraphFor(unknown),
});
expect(result.question).not.toContain("). is true?");
expect(result.question).toBe(
"What would clarify magnitude and nature of cash outflows (operating expenses) in this situation?",
);
});
}); });
@@ -1,7 +1,13 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import { formulateQuestion } from "@/lib/graph/question-formulator.js"; import {
formulateQuestion,
formulateTieResolutionQuestion,
} from "@/lib/graph/question-formulator.js";
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js"; import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
import { explainUnknownSelection } from "@/lib/graph/utils.js"; import {
explainUnknownSelection,
selectActiveUnknownCandidate,
} from "@/lib/graph/utils.js";
function buildLiveShapedGraph() { function buildLiveShapedGraph() {
const summary = makeNode({ const summary = makeNode({
@@ -184,37 +190,57 @@ function neutraliseUnknownWording(graph) {
} }
describe("selection influence diagnostic", () => { describe("selection influence diagnostic", () => {
it("records ordering changes for live-shaped, structure-only, and wording-neutralised fixtures", () => { it("records ambiguous ordering changes for live-shaped, structure-only, and wording-neutralised fixtures", () => {
const liveGraph = buildLiveShapedGraph(); const liveGraph = buildLiveShapedGraph();
const liveExplanation = explainUnknownSelection(liveGraph, []); const liveExplanation = explainUnknownSelection(liveGraph, []);
const liveWinner = liveGraph.nodes.find( const liveSelection = selectActiveUnknownCandidate(liveGraph, []);
(node) => node.id === liveExplanation.selectedNodeId, const tieQuestion = formulateTieResolutionQuestion({ graph: liveGraph });
);
const liveQuestion = formulateQuestion({
node: liveWinner,
graph: liveGraph,
});
const noLinksExplanation = explainUnknownSelection( const noLinksExplanation = explainUnknownSelection(
removeDependencyLinks(liveGraph), removeDependencyLinks(liveGraph),
[], [],
); );
const noLinksSelection = selectActiveUnknownCandidate(
removeDependencyLinks(liveGraph),
[],
);
const neutralWordingExplanation = explainUnknownSelection( const neutralWordingExplanation = explainUnknownSelection(
neutraliseUnknownWording(liveGraph), neutraliseUnknownWording(liveGraph),
[], [],
); );
const neutralSelection = selectActiveUnknownCandidate(
neutraliseUnknownWording(liveGraph),
[],
);
const fallbackQuestion = formulateQuestion({
node: liveGraph.nodes.find((node) => node.id === "nqdzobz"),
graph: liveGraph,
});
const diagnosticRecord = { const diagnosticRecord = {
liveStatus: liveExplanation.status,
liveShapedCandidateOrdering: orderCandidates(liveExplanation), liveShapedCandidateOrdering: orderCandidates(liveExplanation),
liveTiedCandidateIds: liveExplanation.tiedCandidateIds,
noLinksCandidateOrdering: orderCandidates(noLinksExplanation), noLinksCandidateOrdering: orderCandidates(noLinksExplanation),
noLinksStatus: noLinksExplanation.status,
neutralWordingCandidateOrdering: orderCandidates( neutralWordingCandidateOrdering: orderCandidates(
neutralWordingExplanation, neutralWordingExplanation,
), ),
selectedExplanationContributions: neutralStatus: neutralWordingExplanation.status,
liveExplanation.selected?.contributions ?? [], selectedExplanationContributions: liveExplanation.selected?.contributions,
selectedInvestigationStrategy: liveQuestion.strategy, tieQuestion: tieQuestion.question,
liveSelection,
noLinksSelection,
neutralSelection,
fallbackQuestion,
}; };
expect(diagnosticRecord.liveStatus).toBe("ambiguous");
expect(diagnosticRecord.liveTiedCandidateIds).toEqual([
"nqdzobz",
"niewza",
]);
expect(diagnosticRecord.liveShapedCandidateOrdering).toEqual([ expect(diagnosticRecord.liveShapedCandidateOrdering).toEqual([
{ {
nodeId: "nqdzobz", nodeId: "nqdzobz",
@@ -233,9 +259,17 @@ describe("selection influence diagnostic", () => {
unresolvedParentUnknownCount: 0, unresolvedParentUnknownCount: 0,
}, },
]); ]);
expect(diagnosticRecord.liveSelection).toMatchObject({
selectedNode: null,
status: "ambiguous",
tieType: "complete_unresolved_tie",
tiedCandidateIds: ["nqdzobz", "niewza"],
});
expect(diagnosticRecord.noLinksCandidateOrdering).toEqual( expect(diagnosticRecord.noLinksCandidateOrdering).toEqual(
diagnosticRecord.liveShapedCandidateOrdering, diagnosticRecord.liveShapedCandidateOrdering,
); );
expect(diagnosticRecord.noLinksStatus).toBe("ambiguous");
expect(diagnosticRecord.noLinksSelection.status).toBe("ambiguous");
expect(diagnosticRecord.neutralWordingCandidateOrdering).toEqual([ expect(diagnosticRecord.neutralWordingCandidateOrdering).toEqual([
{ {
nodeId: "niewza", nodeId: "niewza",
@@ -252,14 +286,18 @@ describe("selection influence diagnostic", () => {
unresolvedParentUnknownCount: 0, unresolvedParentUnknownCount: 0,
}, },
]); ]);
expect(diagnosticRecord.selectedExplanationContributions).toEqual([ expect(diagnosticRecord.neutralStatus).toBe("ambiguous");
{ expect(diagnosticRecord.neutralSelection.status).toBe("ambiguous");
rule: "downstream_dependencies", expect(diagnosticRecord.selectedExplanationContributions).toBeUndefined();
value: 0, expect(diagnosticRecord.tieQuestion).toBe(
weight: 4, "What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?",
delta: 0, );
}, expect(diagnosticRecord.tieQuestion.toLowerCase()).not.toMatch(
]); /accounts receivable|capex|debt repayments|working capital/,
expect(diagnosticRecord.selectedInvestigationStrategy).toBe("definition"); );
expect(diagnosticRecord.fallbackQuestion.strategy).toBeNull();
expect(diagnosticRecord.fallbackQuestion.question).toBe(
"What would clarify magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts) in this situation?",
);
}); });
}); });
+67
View File
@@ -456,6 +456,7 @@ describe("selectActiveUnknownCandidate", () => {
const result = selectActiveUnknownCandidate(graph, []); const result = selectActiveUnknownCandidate(graph, []);
expect(result.nodeId).toBe("unknown-a"); // Has more dependents (score 2 vs 0) expect(result.nodeId).toBe("unknown-a"); // Has more dependents (score 2 vs 0)
expect(result.status).toBe("selected");
}); });
it("returns one candidate (not array)", () => { it("returns one candidate (not array)", () => {
@@ -619,6 +620,72 @@ describe("selectActiveUnknownCandidate", () => {
const childScore = scoreUnknownCandidate(graph, childUnknown, []); const childScore = scoreUnknownCandidate(graph, childUnknown, []);
expect(parentScore.score).toBeGreaterThan(childScore.score); expect(parentScore.score).toBeGreaterThan(childScore.score);
}); });
it("returns ambiguous for a complete unresolved tie instead of label-based winner", () => {
const unknownA = makeNode({
id: "tie-a",
label: "Magnitude and nature of cash outflows",
description: "Magnitude and nature of cash outflows.",
kind: "unknown",
status: "unknown",
confidence: "high",
});
const unknownB = makeNode({
id: "tie-b",
label:
"Whether revenue recognition timing differs from cash collection timing",
description:
"Whether revenue recognition timing differs from cash collection timing.",
kind: "unknown",
status: "unknown",
confidence: "high",
});
const graph = makeGraph({
centralStatement:
"Revenue increased by 18%, but cash in the bank fell over the same period.",
nodes: [unknownA, unknownB],
edges: [],
activeUnknownNodeId: null,
resolvedNodeIds: [],
currentSummary: "Tie case",
});
const result = selectActiveUnknownCandidate(graph, []);
expect(result).toMatchObject({
selectedNode: null,
status: "ambiguous",
tieType: "complete_unresolved_tie",
tiedCandidateIds: ["tie-a", "tie-b"],
});
expect(result.nodeId).toBeUndefined();
});
it("alphabetical renaming does not resolve a complete tie", () => {
const unknownA = makeNode({
id: "tie-a",
label: "Unknown B",
description: "Unknown factor one.",
kind: "unknown",
});
const unknownB = makeNode({
id: "tie-b",
label: "Unknown A",
description: "Unknown factor two.",
kind: "unknown",
});
const graph = makeGraph({
centralStatement: "Two conflicting signals remain unresolved.",
nodes: [unknownA, unknownB],
edges: [],
activeUnknownNodeId: null,
resolvedNodeIds: [],
currentSummary: "Tie case",
});
const result = selectActiveUnknownCandidate(graph, []);
expect(result.status).toBe("ambiguous");
expect(result.tiedCandidateIds.sort()).toEqual(["tie-a", "tie-b"]);
});
}); });
describe("applyGraphUpdate", () => { describe("applyGraphUpdate", () => {