From a1f6d0c2b9127ae94cd1df3cce448e87bcf8d75c Mon Sep 17 00:00:00 2001 From: robbond Date: Sun, 2 Aug 2026 15:40:06 +0100 Subject: [PATCH] test: inspect structural influence in unknown selection --- docs/v0.6-selection-influence-experiment.md | 50 ++++ .../selection-influence-diagnostic.test.js | 265 ++++++++++++++++++ 2 files changed, 315 insertions(+) create mode 100644 docs/v0.6-selection-influence-experiment.md create mode 100644 tests/graph/selection-influence-diagnostic.test.js diff --git a/docs/v0.6-selection-influence-experiment.md b/docs/v0.6-selection-influence-experiment.md new file mode 100644 index 0000000..6d1dfbf --- /dev/null +++ b/docs/v0.6-selection-influence-experiment.md @@ -0,0 +1,50 @@ +# v0.6 Selection Influence Experiment + +## Hypothesis + +The initial unknown selected for the revenue-versus-cash scenario may be driven more by graph structure, more by semantic keyword matches, or by both together. + +## Scenario + +`Revenue increased by 18%, but cash in the bank fell over the same period.` + +## Actual selected node + +- Node ID: `nqdzobz` +- Label: `Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).` +- Deterministic investigation strategy: `definition` +- Deterministic question: `What evidence would resolve whether magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts). is true?` + +## Structural contribution + +- Downstream dependency count: `0` +- Prerequisite position: no unresolved prerequisites; count `0` +- Dependency ordering / centrality: no candidate had downstream dependants or dependency depth advantage in the live graph + +## Semantic contribution + +- Objective: false +- Actor: false +- Criteria: false +- Measurement: false +- Terminology: false +- Constraint: false +- Pricing: false +- Implementation: false +- Optimisation: false +- Speculative: false +- Contribution list: only `downstream_dependencies` was present, with delta `0` + +## Counterfactual results + +- Live-shaped ordering: `nqdzobz` ranked above `niewza`, but both had score `0`, downstream `0`, and unresolved prerequisites `0` +- Links removed: ordering stayed the same, because the live graph already provided no differentiating structure between the two unknowns +- Wording neutralised: ordering flipped to the first unknown by neutral label order (`Unknown A` before `Unknown B`), showing the outcome remained tie-break-driven rather than structure-driven + +## Conclusion + +For this scenario, the actual winner was not selected because of graph structure and not selected because of semantic keyword weights. The live diagnostics show a complete tie on score, downstream influence, and prerequisite position, with every semantic match category false for both candidates. The winner was therefore chosen by the final tie-break rule, `label_asc`. + +## Is a scoring change justified? + +Not from this single experiment alone. The result shows a diagnostic gap for this scenario, but this task does not justify a scoring change by itself, and no scoring change is made. diff --git a/tests/graph/selection-influence-diagnostic.test.js b/tests/graph/selection-influence-diagnostic.test.js new file mode 100644 index 0000000..a9ac645 --- /dev/null +++ b/tests/graph/selection-influence-diagnostic.test.js @@ -0,0 +1,265 @@ +import { describe, expect, it } from "vitest"; +import { formulateQuestion } from "@/lib/graph/question-formulator.js"; +import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js"; +import { explainUnknownSelection } from "@/lib/graph/utils.js"; + +function buildLiveShapedGraph() { + const summary = makeNode({ + id: "nnvog0y", + label: + "Revenue grew by 18% while corporate cash reserves declined over an identical time frame.", + description: "Summary of the situation from the scenario text", + kind: "state", + status: "provisional", + confidence: "medium", + }); + const revenueObservation = makeNode({ + id: "nri36w9", + label: "Revenue increased by 18%.", + description: "Revenue increased by 18%.", + kind: "observation", + status: "supported", + confidence: "high", + evidenceIds: ["obs_rev"], + }); + const cashObservation = makeNode({ + id: "nnfc48j", + label: "Cash in the bank decreased over the same period.", + description: "Cash in the bank decreased over the same period.", + kind: "observation", + status: "supported", + confidence: "high", + evidenceIds: ["obs_cash"], + }); + const revenueMetric = makeNode({ + id: "nhsd6d5", + label: "Revenue metric (typically accrual-based income statement figure)", + description: + "Revenue metric (typically accrual-based income statement figure)", + kind: "metric", + status: "known", + confidence: "high", + }); + const cashMetric = makeNode({ + id: "neh5m6m", + label: + "Cash balance (liquidity measure on the balance sheet or cash flow statement)", + description: + "Cash balance (liquidity measure on the balance sheet or cash flow statement)", + kind: "metric", + status: "known", + confidence: "high", + }); + const directionalRelationship = makeNode({ + id: "nwo6070", + label: + "Divergent directional movement between top-line revenue growth and net cash position contraction.", + description: + "Divergent directional movement between top-line revenue growth and net cash position contraction.", + kind: "relationship", + status: "supported", + confidence: "high", + }); + const contradictionRelationship = makeNode({ + id: "nuiab02", + label: + "Apparent contradiction between profitability/revenue expansion and liquidity reduction.", + description: + "Apparent contradiction between profitability/revenue expansion and liquidity reduction.", + kind: "relationship", + status: "supported", + confidence: "medium", + }); + const cashTiming = makeNode({ + id: "niewza", + label: + "Whether revenue recognition timing differs from cash collection timing.", + description: + "Whether revenue recognition timing differs from cash collection timing.", + kind: "unknown", + status: "unknown", + confidence: "high", + }); + const cashOutflows = makeNode({ + id: "nqdzobz", + label: + "Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).", + description: + "Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).", + kind: "unknown", + status: "unknown", + confidence: "high", + }); + + const edges = [ + makeEdge({ + id: "e-revenue-summary", + fromNodeId: revenueObservation.id, + toNodeId: summary.id, + relationship: "supports", + description: "Revenue increase supports the scenario summary.", + }), + makeEdge({ + id: "e-cash-summary", + fromNodeId: cashObservation.id, + toNodeId: summary.id, + relationship: "supports", + description: "Cash decline supports the scenario summary.", + }), + makeEdge({ + id: "e-unk-niewza", + fromNodeId: cashTiming.id, + toNodeId: summary.id, + relationship: "depends_on", + description: + "Whether revenue recognition timing differs from cash collection timing. is an unresolved factor for this situation", + }), + makeEdge({ + id: "e-unk-nqdzobz", + fromNodeId: cashOutflows.id, + toNodeId: summary.id, + relationship: "depends_on", + description: + "Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts). is an unresolved factor for this situation", + }), + ]; + + return makeGraph({ + centralStatement: + "Revenue increased by 18%, but cash in the bank fell over the same period.", + nodes: [ + summary, + revenueObservation, + cashObservation, + revenueMetric, + cashMetric, + directionalRelationship, + contradictionRelationship, + cashTiming, + cashOutflows, + ], + edges, + activeUnknownNodeId: null, + resolvedNodeIds: [], + currentSummary: "Diagnostic selection influence fixture", + }); +} + +function orderCandidates(explanation) { + return explanation.candidates.map((candidate) => ({ + nodeId: candidate.nodeId, + label: candidate.label, + score: candidate.score, + downstreamCount: candidate.downstreamCount, + unresolvedParentUnknownCount: candidate.unresolvedParentUnknownCount, + })); +} + +function removeDependencyLinks(graph) { + const nodes = graph.nodes.map((node) => ({ + ...node, + dependsOn: [], + affects: [], + parentId: null, + childIds: [], + })); + const edges = (graph.edges || []).filter( + (edge) => edge.relationship !== "depends_on", + ); + return makeGraph({ ...graph, nodes, edges, activeUnknownNodeId: null }); +} + +function neutraliseUnknownWording(graph) { + let counter = 0; + const nodes = graph.nodes.map((node) => { + if (node.kind !== "unknown") return { ...node }; + counter += 1; + return { + ...node, + label: `Unknown ${String.fromCharCode(64 + counter)}`, + description: `Unknown factor ${counter} relevant to the scenario.`, + }; + }); + return makeGraph({ ...graph, nodes, activeUnknownNodeId: null }); +} + +describe("selection influence diagnostic", () => { + it("records ordering changes for live-shaped, structure-only, and wording-neutralised fixtures", () => { + const liveGraph = buildLiveShapedGraph(); + const liveExplanation = explainUnknownSelection(liveGraph, []); + const liveWinner = liveGraph.nodes.find( + (node) => node.id === liveExplanation.selectedNodeId, + ); + const liveQuestion = formulateQuestion({ + node: liveWinner, + graph: liveGraph, + }); + + const noLinksExplanation = explainUnknownSelection( + removeDependencyLinks(liveGraph), + [], + ); + const neutralWordingExplanation = explainUnknownSelection( + neutraliseUnknownWording(liveGraph), + [], + ); + + const diagnosticRecord = { + liveShapedCandidateOrdering: orderCandidates(liveExplanation), + noLinksCandidateOrdering: orderCandidates(noLinksExplanation), + neutralWordingCandidateOrdering: orderCandidates( + neutralWordingExplanation, + ), + selectedExplanationContributions: + liveExplanation.selected?.contributions ?? [], + selectedInvestigationStrategy: liveQuestion.strategy, + }; + + expect(diagnosticRecord.liveShapedCandidateOrdering).toEqual([ + { + nodeId: "nqdzobz", + label: + "Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).", + score: 0, + downstreamCount: 0, + unresolvedParentUnknownCount: 0, + }, + { + nodeId: "niewza", + label: + "Whether revenue recognition timing differs from cash collection timing.", + score: 0, + downstreamCount: 0, + unresolvedParentUnknownCount: 0, + }, + ]); + expect(diagnosticRecord.noLinksCandidateOrdering).toEqual( + diagnosticRecord.liveShapedCandidateOrdering, + ); + expect(diagnosticRecord.neutralWordingCandidateOrdering).toEqual([ + { + nodeId: "niewza", + label: "Unknown A", + score: 0, + downstreamCount: 0, + unresolvedParentUnknownCount: 0, + }, + { + nodeId: "nqdzobz", + label: "Unknown B", + score: 0, + downstreamCount: 0, + unresolvedParentUnknownCount: 0, + }, + ]); + expect(diagnosticRecord.selectedExplanationContributions).toEqual([ + { + rule: "downstream_dependencies", + value: 0, + weight: 4, + delta: 0, + }, + ]); + expect(diagnosticRecord.selectedInvestigationStrategy).toBe("definition"); + }); +});