test: inspect structural influence in unknown selection
This commit is contained in:
@@ -0,0 +1,50 @@
|
||||
# v0.6 Selection Influence Experiment
|
||||
|
||||
## Hypothesis
|
||||
|
||||
The initial unknown selected for the revenue-versus-cash scenario may be driven more by graph structure, more by semantic keyword matches, or by both together.
|
||||
|
||||
## Scenario
|
||||
|
||||
`Revenue increased by 18%, but cash in the bank fell over the same period.`
|
||||
|
||||
## Actual selected node
|
||||
|
||||
- Node ID: `nqdzobz`
|
||||
- Label: `Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).`
|
||||
- Deterministic investigation strategy: `definition`
|
||||
- Deterministic question: `What evidence would resolve whether magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts). is true?`
|
||||
|
||||
## Structural contribution
|
||||
|
||||
- Downstream dependency count: `0`
|
||||
- Prerequisite position: no unresolved prerequisites; count `0`
|
||||
- Dependency ordering / centrality: no candidate had downstream dependants or dependency depth advantage in the live graph
|
||||
|
||||
## Semantic contribution
|
||||
|
||||
- Objective: false
|
||||
- Actor: false
|
||||
- Criteria: false
|
||||
- Measurement: false
|
||||
- Terminology: false
|
||||
- Constraint: false
|
||||
- Pricing: false
|
||||
- Implementation: false
|
||||
- Optimisation: false
|
||||
- Speculative: false
|
||||
- Contribution list: only `downstream_dependencies` was present, with delta `0`
|
||||
|
||||
## Counterfactual results
|
||||
|
||||
- Live-shaped ordering: `nqdzobz` ranked above `niewza`, but both had score `0`, downstream `0`, and unresolved prerequisites `0`
|
||||
- Links removed: ordering stayed the same, because the live graph already provided no differentiating structure between the two unknowns
|
||||
- Wording neutralised: ordering flipped to the first unknown by neutral label order (`Unknown A` before `Unknown B`), showing the outcome remained tie-break-driven rather than structure-driven
|
||||
|
||||
## Conclusion
|
||||
|
||||
For this scenario, the actual winner was not selected because of graph structure and not selected because of semantic keyword weights. The live diagnostics show a complete tie on score, downstream influence, and prerequisite position, with every semantic match category false for both candidates. The winner was therefore chosen by the final tie-break rule, `label_asc`.
|
||||
|
||||
## Is a scoring change justified?
|
||||
|
||||
Not from this single experiment alone. The result shows a diagnostic gap for this scenario, but this task does not justify a scoring change by itself, and no scoring change is made.
|
||||
@@ -0,0 +1,265 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { formulateQuestion } from "@/lib/graph/question-formulator.js";
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
import { explainUnknownSelection } from "@/lib/graph/utils.js";
|
||||
|
||||
function buildLiveShapedGraph() {
|
||||
const summary = makeNode({
|
||||
id: "nnvog0y",
|
||||
label:
|
||||
"Revenue grew by 18% while corporate cash reserves declined over an identical time frame.",
|
||||
description: "Summary of the situation from the scenario text",
|
||||
kind: "state",
|
||||
status: "provisional",
|
||||
confidence: "medium",
|
||||
});
|
||||
const revenueObservation = makeNode({
|
||||
id: "nri36w9",
|
||||
label: "Revenue increased by 18%.",
|
||||
description: "Revenue increased by 18%.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
evidenceIds: ["obs_rev"],
|
||||
});
|
||||
const cashObservation = makeNode({
|
||||
id: "nnfc48j",
|
||||
label: "Cash in the bank decreased over the same period.",
|
||||
description: "Cash in the bank decreased over the same period.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
evidenceIds: ["obs_cash"],
|
||||
});
|
||||
const revenueMetric = makeNode({
|
||||
id: "nhsd6d5",
|
||||
label: "Revenue metric (typically accrual-based income statement figure)",
|
||||
description:
|
||||
"Revenue metric (typically accrual-based income statement figure)",
|
||||
kind: "metric",
|
||||
status: "known",
|
||||
confidence: "high",
|
||||
});
|
||||
const cashMetric = makeNode({
|
||||
id: "neh5m6m",
|
||||
label:
|
||||
"Cash balance (liquidity measure on the balance sheet or cash flow statement)",
|
||||
description:
|
||||
"Cash balance (liquidity measure on the balance sheet or cash flow statement)",
|
||||
kind: "metric",
|
||||
status: "known",
|
||||
confidence: "high",
|
||||
});
|
||||
const directionalRelationship = makeNode({
|
||||
id: "nwo6070",
|
||||
label:
|
||||
"Divergent directional movement between top-line revenue growth and net cash position contraction.",
|
||||
description:
|
||||
"Divergent directional movement between top-line revenue growth and net cash position contraction.",
|
||||
kind: "relationship",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
});
|
||||
const contradictionRelationship = makeNode({
|
||||
id: "nuiab02",
|
||||
label:
|
||||
"Apparent contradiction between profitability/revenue expansion and liquidity reduction.",
|
||||
description:
|
||||
"Apparent contradiction between profitability/revenue expansion and liquidity reduction.",
|
||||
kind: "relationship",
|
||||
status: "supported",
|
||||
confidence: "medium",
|
||||
});
|
||||
const cashTiming = makeNode({
|
||||
id: "niewza",
|
||||
label:
|
||||
"Whether revenue recognition timing differs from cash collection timing.",
|
||||
description:
|
||||
"Whether revenue recognition timing differs from cash collection timing.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const cashOutflows = makeNode({
|
||||
id: "nqdzobz",
|
||||
label:
|
||||
"Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).",
|
||||
description:
|
||||
"Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
|
||||
const edges = [
|
||||
makeEdge({
|
||||
id: "e-revenue-summary",
|
||||
fromNodeId: revenueObservation.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "supports",
|
||||
description: "Revenue increase supports the scenario summary.",
|
||||
}),
|
||||
makeEdge({
|
||||
id: "e-cash-summary",
|
||||
fromNodeId: cashObservation.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "supports",
|
||||
description: "Cash decline supports the scenario summary.",
|
||||
}),
|
||||
makeEdge({
|
||||
id: "e-unk-niewza",
|
||||
fromNodeId: cashTiming.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "depends_on",
|
||||
description:
|
||||
"Whether revenue recognition timing differs from cash collection timing. is an unresolved factor for this situation",
|
||||
}),
|
||||
makeEdge({
|
||||
id: "e-unk-nqdzobz",
|
||||
fromNodeId: cashOutflows.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "depends_on",
|
||||
description:
|
||||
"Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts). is an unresolved factor for this situation",
|
||||
}),
|
||||
];
|
||||
|
||||
return makeGraph({
|
||||
centralStatement:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
nodes: [
|
||||
summary,
|
||||
revenueObservation,
|
||||
cashObservation,
|
||||
revenueMetric,
|
||||
cashMetric,
|
||||
directionalRelationship,
|
||||
contradictionRelationship,
|
||||
cashTiming,
|
||||
cashOutflows,
|
||||
],
|
||||
edges,
|
||||
activeUnknownNodeId: null,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Diagnostic selection influence fixture",
|
||||
});
|
||||
}
|
||||
|
||||
function orderCandidates(explanation) {
|
||||
return explanation.candidates.map((candidate) => ({
|
||||
nodeId: candidate.nodeId,
|
||||
label: candidate.label,
|
||||
score: candidate.score,
|
||||
downstreamCount: candidate.downstreamCount,
|
||||
unresolvedParentUnknownCount: candidate.unresolvedParentUnknownCount,
|
||||
}));
|
||||
}
|
||||
|
||||
function removeDependencyLinks(graph) {
|
||||
const nodes = graph.nodes.map((node) => ({
|
||||
...node,
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
}));
|
||||
const edges = (graph.edges || []).filter(
|
||||
(edge) => edge.relationship !== "depends_on",
|
||||
);
|
||||
return makeGraph({ ...graph, nodes, edges, activeUnknownNodeId: null });
|
||||
}
|
||||
|
||||
function neutraliseUnknownWording(graph) {
|
||||
let counter = 0;
|
||||
const nodes = graph.nodes.map((node) => {
|
||||
if (node.kind !== "unknown") return { ...node };
|
||||
counter += 1;
|
||||
return {
|
||||
...node,
|
||||
label: `Unknown ${String.fromCharCode(64 + counter)}`,
|
||||
description: `Unknown factor ${counter} relevant to the scenario.`,
|
||||
};
|
||||
});
|
||||
return makeGraph({ ...graph, nodes, activeUnknownNodeId: null });
|
||||
}
|
||||
|
||||
describe("selection influence diagnostic", () => {
|
||||
it("records ordering changes for live-shaped, structure-only, and wording-neutralised fixtures", () => {
|
||||
const liveGraph = buildLiveShapedGraph();
|
||||
const liveExplanation = explainUnknownSelection(liveGraph, []);
|
||||
const liveWinner = liveGraph.nodes.find(
|
||||
(node) => node.id === liveExplanation.selectedNodeId,
|
||||
);
|
||||
const liveQuestion = formulateQuestion({
|
||||
node: liveWinner,
|
||||
graph: liveGraph,
|
||||
});
|
||||
|
||||
const noLinksExplanation = explainUnknownSelection(
|
||||
removeDependencyLinks(liveGraph),
|
||||
[],
|
||||
);
|
||||
const neutralWordingExplanation = explainUnknownSelection(
|
||||
neutraliseUnknownWording(liveGraph),
|
||||
[],
|
||||
);
|
||||
|
||||
const diagnosticRecord = {
|
||||
liveShapedCandidateOrdering: orderCandidates(liveExplanation),
|
||||
noLinksCandidateOrdering: orderCandidates(noLinksExplanation),
|
||||
neutralWordingCandidateOrdering: orderCandidates(
|
||||
neutralWordingExplanation,
|
||||
),
|
||||
selectedExplanationContributions:
|
||||
liveExplanation.selected?.contributions ?? [],
|
||||
selectedInvestigationStrategy: liveQuestion.strategy,
|
||||
};
|
||||
|
||||
expect(diagnosticRecord.liveShapedCandidateOrdering).toEqual([
|
||||
{
|
||||
nodeId: "nqdzobz",
|
||||
label:
|
||||
"Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).",
|
||||
score: 0,
|
||||
downstreamCount: 0,
|
||||
unresolvedParentUnknownCount: 0,
|
||||
},
|
||||
{
|
||||
nodeId: "niewza",
|
||||
label:
|
||||
"Whether revenue recognition timing differs from cash collection timing.",
|
||||
score: 0,
|
||||
downstreamCount: 0,
|
||||
unresolvedParentUnknownCount: 0,
|
||||
},
|
||||
]);
|
||||
expect(diagnosticRecord.noLinksCandidateOrdering).toEqual(
|
||||
diagnosticRecord.liveShapedCandidateOrdering,
|
||||
);
|
||||
expect(diagnosticRecord.neutralWordingCandidateOrdering).toEqual([
|
||||
{
|
||||
nodeId: "niewza",
|
||||
label: "Unknown A",
|
||||
score: 0,
|
||||
downstreamCount: 0,
|
||||
unresolvedParentUnknownCount: 0,
|
||||
},
|
||||
{
|
||||
nodeId: "nqdzobz",
|
||||
label: "Unknown B",
|
||||
score: 0,
|
||||
downstreamCount: 0,
|
||||
unresolvedParentUnknownCount: 0,
|
||||
},
|
||||
]);
|
||||
expect(diagnosticRecord.selectedExplanationContributions).toEqual([
|
||||
{
|
||||
rule: "downstream_dependencies",
|
||||
value: 0,
|
||||
weight: 4,
|
||||
delta: 0,
|
||||
},
|
||||
]);
|
||||
expect(diagnosticRecord.selectedInvestigationStrategy).toBe("definition");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user