test: generalise question priority across decisions

This commit is contained in:
2026-08-02 13:02:49 +01:00
parent 48ce66dddb
commit 781d6a462f
4 changed files with 675 additions and 8 deletions
@@ -0,0 +1,48 @@
# v0.5 Question Priority Generalisation
## Hypothesis
The current deterministic unknown selector and graph-context question formulator should generalise across several decision types by selecting a foundational unknown before downstream implementation or pricing leaves.
## Scenarios
1. Should we hire another engineer?
2. Should we replace the delivery vans?
3. Should we launch in another country?
4. Should we continue a project that is over budget?
5. Should we introduce a paid support tier?
## Results
| Scenario | Selected unknown | Strategy | Pass/Fail |
| ---------------------------- | --------------------------- | -------------------- | --------- |
| Hire another engineer | `hire-success-criteria` | `decision criterion` | Pass |
| Replace the delivery vans | `van-reliability-threshold` | `decision criterion` | Pass |
| Launch in another country | `country-value-threshold` | `actor/customer` | Pass |
| Continue over-budget project | `project-benefit-threshold` | `decision criterion` | Pass |
| Introduce paid support tier | `support-value-threshold` | `actor/customer` | Pass |
## Repeated failure patterns
Two repeated structural formulation failures appeared before the final pass:
1. **Constraint language in surrounding graph context outranked node-local decision-threshold language** in more than one case.
2. **Baseline language in surrounding graph context outranked node-local threshold language** in more than one case.
Both failures affected formulation strategy, not deterministic unknown selection.
## Code change made
A small deterministic change was made in `lib/graph/question-formulator.js`:
- prefer node-local `definition` language before broader criterion inference
- prefer node-local `decision criterion` language before context-only `constraint` inference
- only treat `baseline` or `constraint` as primary when the selected node itself carries that language, otherwise allow them as fallback strategies later
No architecture, UI, persistence, prompt, scoring, additional model turns, or provider calls were added.
## Remaining limitations
- In two passing cases, the selector chose a threshold-style foundational node while the formulator still used an `actor/customer` strategy because related context strongly referenced customers or recipients.
- This experiment is fixture-driven and deterministic; it is useful for regression protection, not scientific validation.
- The suite exercises the production path without model calls, but it does not prove behaviour over arbitrary real-world graph structures.
+39 -8
View File
@@ -141,6 +141,9 @@ function toGerundPhrase(phrase) {
function detectStrategy({ node, graph, relatedNodes, combinedText, meaning }) {
const text = normaliseText(combinedText);
const nodeText = normaliseText(
`${node?.label || ""} ${node?.description || ""}`,
);
const relatedText = normaliseText(
relatedNodes
.map((relatedNode) => `${relatedNode.label} ${relatedNode.description}`)
@@ -160,19 +163,25 @@ function detectStrategy({ node, graph, relatedNodes, combinedText, meaning }) {
`${text} ${relatedText} ${resolvedValues.join(" ")}`,
) || Boolean(actionPhrase);
if (
const hasConstraintLanguage =
/\b(constraint|limit|budget|deadline|requirement|regulation|capacity)\b/.test(
text,
)
) {
return { strategy: "constraint", meaning, actionPhrase };
}
);
const hasPrimaryConstraintLanguage =
/\b(constraint|limit|budget|deadline|requirement|regulation|capacity)\b/.test(
nodeText,
);
if (/\b(customer|user|buyer|stakeholder|recipient|audience)\b/.test(text)) {
return { strategy: "actor/customer", meaning, actionPhrase };
}
if (/\b(before|previous|baseline|prior|comparable state)\b/.test(text)) {
const hasBaselineLanguage =
/\b(before|previous|baseline|prior|comparable state)\b/.test(text);
const hasPrimaryBaselineLanguage =
/\b(before|previous|baseline|prior|comparable state)\b/.test(nodeText);
if (hasBaselineLanguage && hasPrimaryBaselineLanguage) {
return { strategy: "baseline", meaning, actionPhrase };
}
@@ -182,6 +191,12 @@ function detectStrategy({ node, graph, relatedNodes, combinedText, meaning }) {
const hasDefinitionLanguage =
/\b(define|definition|meaning|term|terminology)\b/.test(text);
const hasPrimaryDefinitionLanguage =
/\b(define|definition|meaning|term|terminology)\b/.test(nodeText);
const hasCriteriaLanguage =
/\b(success criteria|success threshold|threshold|decision criteria|criterion|justify|sufficient)\b/.test(
nodeText,
);
const hasDecisionValueLanguage =
decisionContext &&
/\b(value|commercial value|commercial viability|viability|justify|sufficient|success|threshold|criterion)\b/.test(
@@ -196,14 +211,30 @@ function detectStrategy({ node, graph, relatedNodes, combinedText, meaning }) {
return { strategy: "measurement", meaning, actionPhrase };
}
if (/\b(define|definition|meaning|term|terminology)\b/.test(text)) {
if (hasPrimaryDefinitionLanguage) {
return { strategy: "definition", meaning, actionPhrase };
}
if (hasDecisionValueLanguage) {
if (hasDecisionValueLanguage || hasCriteriaLanguage) {
return { strategy: "decision criterion", meaning, actionPhrase };
}
if (hasConstraintLanguage && hasPrimaryConstraintLanguage) {
return { strategy: "constraint", meaning, actionPhrase };
}
if (hasDefinitionLanguage) {
return { strategy: "definition", meaning, actionPhrase };
}
if (hasBaselineLanguage) {
return { strategy: "baseline", meaning, actionPhrase };
}
if (hasConstraintLanguage) {
return { strategy: "constraint", meaning, actionPhrase };
}
if (/\b(evidence|proof|validate|validation|signal|demand)\b/.test(text)) {
return { strategy: "evidence", meaning, actionPhrase };
}
+433
View File
@@ -0,0 +1,433 @@
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
function makeScenarioGraph({
scenario,
decisionNode,
answeredContextUnknown,
foundationalUnknown,
consequentialUnknown,
downstreamLeaf,
}) {
const nodes = [
decisionNode,
answeredContextUnknown,
foundationalUnknown,
consequentialUnknown,
downstreamLeaf,
];
const edges = [
makeEdge({
id: `${decisionNode.id}-to-${foundationalUnknown.id}`,
fromNodeId: decisionNode.id,
toNodeId: foundationalUnknown.id,
relationship: "depends_on",
description: `${decisionNode.label} depends on ${foundationalUnknown.label}.`,
}),
makeEdge({
id: `${answeredContextUnknown.id}-to-${consequentialUnknown.id}`,
fromNodeId: answeredContextUnknown.id,
toNodeId: consequentialUnknown.id,
relationship: "depends_on",
description: `${consequentialUnknown.label} was surfaced from resolved context.`,
}),
makeEdge({
id: `${foundationalUnknown.id}-to-${consequentialUnknown.id}`,
fromNodeId: foundationalUnknown.id,
toNodeId: consequentialUnknown.id,
relationship: "depends_on",
description: `${consequentialUnknown.label} depends on ${foundationalUnknown.label}.`,
}),
makeEdge({
id: `${consequentialUnknown.id}-to-${downstreamLeaf.id}`,
fromNodeId: consequentialUnknown.id,
toNodeId: downstreamLeaf.id,
relationship: "depends_on",
description: `${downstreamLeaf.label} depends on ${consequentialUnknown.label}.`,
}),
];
return makeGraph({
centralStatement: scenario,
nodes,
edges,
activeUnknownNodeId: answeredContextUnknown.id,
resolvedNodeIds: [],
currentSummary: "Generalisation fixture graph",
});
}
export const questionPriorityGeneralisationFixtures = [
{
key: "hire-engineer",
scenario: "Should we hire another engineer?",
decisionType: "resourcing decision",
acceptableFoundationalUnknownNodeIds: [
"hire-success-criteria",
"hire-bottleneck",
],
prohibitedFirstTopics: ["salary", "job advert", "programming language"],
acceptableQuestionStrategies: ["decision criterion", "constraint"],
notes:
"The first question should establish whether more engineering capacity is justified before compensation or implementation details.",
graph: makeScenarioGraph({
scenario: "Should we hire another engineer?",
decisionNode: makeNode({
id: "hire-decision",
label: "Hiring another engineer decision",
description: "Decision about increasing engineering capacity.",
kind: "state",
status: "known",
confidence: "medium",
value: "Deciding whether to hire another engineer",
childIds: ["hire-success-criteria"],
}),
answeredContextUnknown: makeNode({
id: "hire-delays-known",
label: "Delivery delays established",
description:
"Need to confirm whether recent delivery delays are real because this context determines whether a capacity decision is even relevant.",
kind: "unknown",
status: "unknown",
confidence: "high",
value:
"The roadmap is slipping because the current team cannot clear the queue.",
childIds: ["hire-bottleneck"],
}),
foundationalUnknown: makeNode({
id: "hire-success-criteria",
label: "Hiring success threshold",
description:
"Need the success threshold because the hiring decision depends on what improvement would justify adding headcount.",
kind: "unknown",
status: "unknown",
confidence: "high",
parentId: "hire-decision",
childIds: ["hire-bottleneck"],
}),
consequentialUnknown: makeNode({
id: "hire-bottleneck",
label: "Primary delivery bottleneck",
description:
"Need the main bottleneck because the team must know whether another engineer would relieve the limiting constraint.",
kind: "unknown",
status: "unknown",
confidence: "high",
dependsOn: ["hire-success-criteria"],
parentId: "hire-success-criteria",
childIds: ["hire-salary"],
}),
downstreamLeaf: makeNode({
id: "hire-salary",
label: "Engineer salary budget",
description:
"Need the salary range because compensation planning comes after the hiring case is established.",
kind: "unknown",
status: "unknown",
confidence: "medium",
dependsOn: ["hire-bottleneck"],
parentId: "hire-bottleneck",
}),
}),
},
{
key: "replace-vans",
scenario: "Should we replace the delivery vans?",
decisionType: "asset replacement decision",
acceptableFoundationalUnknownNodeIds: [
"van-reliability-threshold",
"van-service-constraint",
],
prohibitedFirstTopics: [
"purchase price",
"paint colour",
"finance provider",
],
acceptableQuestionStrategies: ["decision criterion", "constraint"],
notes:
"The first question should establish whether the fleet is failing a threshold that justifies replacement.",
graph: makeScenarioGraph({
scenario: "Should we replace the delivery vans?",
decisionNode: makeNode({
id: "van-decision",
label: "Replace delivery vans decision",
description: "Decision about replacing the current delivery fleet.",
kind: "state",
status: "known",
confidence: "medium",
value: "Deciding whether to replace the delivery vans",
childIds: ["van-reliability-threshold"],
}),
answeredContextUnknown: makeNode({
id: "van-breakdowns-known",
label: "Breakdown trend confirmed",
description:
"Need to confirm whether the recent rise in breakdowns is real because that context determines whether fleet replacement is relevant.",
kind: "unknown",
status: "unknown",
confidence: "high",
value:
"Breakdowns and missed deliveries have increased over the last quarter.",
childIds: ["van-service-constraint"],
}),
foundationalUnknown: makeNode({
id: "van-reliability-threshold",
label: "Replacement justification threshold",
description:
"Need the threshold because the replacement decision depends on what level of reliability loss is enough to justify replacing the fleet.",
kind: "unknown",
status: "unknown",
confidence: "high",
parentId: "van-decision",
childIds: ["van-service-constraint"],
}),
consequentialUnknown: makeNode({
id: "van-service-constraint",
label: "Operational service constraint",
description:
"Need the limiting service constraint because the team must know how vehicle unreliability is affecting deliveries before comparing purchasing options.",
kind: "unknown",
status: "unknown",
confidence: "high",
dependsOn: ["van-reliability-threshold"],
parentId: "van-reliability-threshold",
childIds: ["van-price"],
}),
downstreamLeaf: makeNode({
id: "van-price",
label: "Exact replacement purchase price",
description:
"Need the exact purchase price because financing analysis comes after replacement is justified.",
kind: "unknown",
status: "unknown",
confidence: "medium",
dependsOn: ["van-service-constraint"],
parentId: "van-service-constraint",
}),
}),
},
{
key: "launch-country",
scenario: "Should we launch in another country?",
decisionType: "market expansion decision",
acceptableFoundationalUnknownNodeIds: [
"country-customer",
"country-value-threshold",
],
prohibitedFirstTopics: [
"launch date",
"office location",
"advertising channel",
],
acceptableQuestionStrategies: ["actor/customer", "decision criterion"],
notes:
"The first question should clarify the customer or value case for expansion before rollout logistics.",
graph: makeScenarioGraph({
scenario: "Should we launch in another country?",
decisionNode: makeNode({
id: "country-decision",
label: "Launch in another country decision",
description: "Decision about entering a new national market.",
kind: "state",
status: "known",
confidence: "medium",
value: "Deciding whether to launch in another country",
childIds: ["country-customer"],
}),
answeredContextUnknown: makeNode({
id: "country-interest-known",
label: "Inbound interest confirmed",
description:
"Need to confirm whether inbound interest from another country is real because that context determines whether expansion is relevant.",
kind: "unknown",
status: "unknown",
confidence: "medium",
value:
"Prospective customers from another country are asking for access.",
childIds: ["country-value-threshold"],
}),
foundationalUnknown: makeNode({
id: "country-customer",
label: "Relevant customer in the new country",
description:
"Need the relevant customer because the expansion decision depends on who experiences the problem or receives the value in that market.",
kind: "unknown",
status: "unknown",
confidence: "high",
parentId: "country-decision",
childIds: ["country-value-threshold"],
}),
consequentialUnknown: makeNode({
id: "country-value-threshold",
label: "Expansion value threshold",
description:
"Need the value threshold because the team must know what evidence of demand or value would justify entering the new country.",
kind: "unknown",
status: "unknown",
confidence: "high",
dependsOn: ["country-customer"],
parentId: "country-customer",
childIds: ["country-launch-date"],
}),
downstreamLeaf: makeNode({
id: "country-launch-date",
label: "Country launch date",
description:
"Need the launch date because rollout planning follows once the expansion case is established.",
kind: "unknown",
status: "unknown",
confidence: "medium",
dependsOn: ["country-value-threshold"],
parentId: "country-value-threshold",
}),
}),
},
{
key: "over-budget-project",
scenario: "Should we continue a project that is over budget?",
decisionType: "continuation decision",
acceptableFoundationalUnknownNodeIds: [
"project-benefit-threshold",
"project-remaining-benefit",
],
prohibitedFirstTopics: ["sunk cost", "project logo", "final launch date"],
acceptableQuestionStrategies: ["decision criterion", "objective"],
notes:
"The first question should establish remaining value or success threshold before sunk-cost framing or launch timing.",
graph: makeScenarioGraph({
scenario: "Should we continue a project that is over budget?",
decisionNode: makeNode({
id: "project-decision",
label: "Continue over-budget project decision",
description:
"Decision about continuing a project that has exceeded budget.",
kind: "state",
status: "known",
confidence: "medium",
value: "Deciding whether to continue the over-budget project",
childIds: ["project-benefit-threshold"],
}),
answeredContextUnknown: makeNode({
id: "project-overrun-known",
label: "Budget overrun confirmed",
description:
"Need to confirm whether the project is materially over budget because that context determines whether a continuation decision is relevant.",
kind: "unknown",
status: "unknown",
confidence: "high",
value: "The project has exceeded its approved budget by 35 percent.",
childIds: ["project-remaining-benefit"],
}),
foundationalUnknown: makeNode({
id: "project-benefit-threshold",
label: "Continuation success threshold",
description:
"Need the threshold because the continuation decision depends on what remaining benefit would still justify completing the project.",
kind: "unknown",
status: "unknown",
confidence: "high",
parentId: "project-decision",
childIds: ["project-remaining-benefit"],
}),
consequentialUnknown: makeNode({
id: "project-remaining-benefit",
label: "Remaining project benefit",
description:
"Need the remaining benefit because the team must know what value is still achievable before deciding whether to continue.",
kind: "unknown",
status: "unknown",
confidence: "high",
dependsOn: ["project-benefit-threshold"],
parentId: "project-benefit-threshold",
childIds: ["project-launch-date"],
}),
downstreamLeaf: makeNode({
id: "project-launch-date",
label: "Final launch date",
description:
"Need the final launch date because scheduling details only matter after remaining value is established.",
kind: "unknown",
status: "unknown",
confidence: "medium",
dependsOn: ["project-remaining-benefit"],
parentId: "project-remaining-benefit",
}),
}),
},
{
key: "paid-support-tier",
scenario: "Should we introduce a paid support tier?",
decisionType: "commercial packaging decision",
acceptableFoundationalUnknownNodeIds: [
"support-customer",
"support-value-threshold",
],
prohibitedFirstTopics: [
"subscription price",
"payment provider",
"tier name",
],
acceptableQuestionStrategies: ["actor/customer", "decision criterion"],
notes:
"The first question should establish who values paid support or what outcome would justify offering it before pricing details.",
graph: makeScenarioGraph({
scenario: "Should we introduce a paid support tier?",
decisionNode: makeNode({
id: "support-decision",
label: "Introduce paid support tier decision",
description: "Decision about adding a paid support offering.",
kind: "state",
status: "known",
confidence: "medium",
value: "Deciding whether to introduce a paid support tier",
childIds: ["support-customer"],
}),
answeredContextUnknown: makeNode({
id: "support-requests-known",
label: "Support request pattern confirmed",
description:
"Need to confirm whether repeated requests for faster support responses are real because that context determines whether a paid tier is relevant.",
kind: "unknown",
status: "unknown",
confidence: "high",
value:
"Some users are asking for guaranteed response times and escalation help.",
childIds: ["support-value-threshold"],
}),
foundationalUnknown: makeNode({
id: "support-customer",
label: "Customer willing to pay for support",
description:
"Need the customer because the decision depends on who experiences enough support pain or receives enough value to pay for a support tier.",
kind: "unknown",
status: "unknown",
confidence: "high",
parentId: "support-decision",
childIds: ["support-value-threshold"],
}),
consequentialUnknown: makeNode({
id: "support-value-threshold",
label: "Paid support value threshold",
description:
"Need the value threshold because the team must know what outcome would justify introducing paid support before setting packaging details.",
kind: "unknown",
status: "unknown",
confidence: "high",
dependsOn: ["support-customer"],
parentId: "support-customer",
childIds: ["support-price"],
}),
downstreamLeaf: makeNode({
id: "support-price",
label: "Support subscription price",
description:
"Need the subscription price because pricing and payment setup come after the support value case is established.",
kind: "unknown",
status: "unknown",
confidence: "medium",
dependsOn: ["support-value-threshold"],
parentId: "support-value-threshold",
}),
}),
},
];
@@ -0,0 +1,155 @@
import { describe, expect, it } from "vitest";
import { applyValidatedProposal } from "@/lib/graph/apply-proposal.js";
import { formulateQuestion } from "@/lib/graph/question-formulator.js";
import { selectActiveUnknownCandidate } from "@/lib/graph/utils.js";
import { questionPriorityGeneralisationFixtures } from "@/tests/fixtures/question-priority-generalisation.js";
function clone(value) {
return JSON.parse(JSON.stringify(value));
}
function buildResolutionProposal(graph) {
const activeNode = graph.nodes.find(
(node) => node.id === graph.activeUnknownNodeId,
);
const placeholderCandidate = graph.nodes.find(
(node) => node.kind === "unknown" && node.id !== activeNode.id,
);
return {
addedNodes: [],
updatedNodes: [
{
nodeId: activeNode.id,
previousStatus: activeNode.status,
newStatus: "resolved",
previousValue: activeNode.value ?? null,
newValue: activeNode.value ?? "Resolved context answer",
reason:
"The resolved context unknown is treated as answered for fixture progression.",
},
],
addedEdges: [],
removedEdgeIds: [],
resolvedUnknownNodeIds: [activeNode.id],
affectedNodeIds: [],
selectedQuestion: {
nodeId: placeholderCandidate?.id,
question: "Placeholder candidate question?",
reason: "Candidate only; deterministic selector should override it.",
},
};
}
function assertQuestionStructure(question) {
expect(question.match(/\?/g) || []).toHaveLength(1);
expect(question).not.toMatch(/\?\s*(and|or)\b/i);
expect(question).not.toMatch(/^What is\s+/i);
expect(question).toMatch(/^(What|Who|When)\b/);
}
describe("question priority generalisation", () => {
for (const fixture of questionPriorityGeneralisationFixtures) {
it(`${fixture.scenario} selects a foundational unknown and singular answerable strategy`, () => {
const originalGraph = clone(fixture.graph);
const deterministicSelection = selectActiveUnknownCandidate(
fixture.graph,
[fixture.graph.activeUnknownNodeId],
);
const result = applyValidatedProposal({
situationGraph: fixture.graph,
proposal: buildResolutionProposal(fixture.graph),
});
expect(result.success).toBe(true);
expect(fixture.graph).toEqual(originalGraph);
expect(result.graphUpdate.selectedQuestion?.question).toBe(
"Placeholder candidate question?",
);
expect(deterministicSelection.nodeId).toBe(
result.selectedQuestion.nodeId,
);
expect(fixture.acceptableFoundationalUnknownNodeIds).toContain(
result.selectedQuestion.nodeId,
);
expect(result.selectedQuestion.nodeId).not.toBe(
fixture.graph.nodes[fixture.graph.nodes.length - 1].id,
);
expect(fixture.acceptableQuestionStrategies).toContain(
result.selectedQuestion.strategy,
);
assertQuestionStructure(result.selectedQuestion.question);
const lowerQuestion = result.selectedQuestion.question.toLowerCase();
for (const topic of fixture.prohibitedFirstTopics) {
expect(lowerQuestion).not.toContain(topic.toLowerCase());
}
const selectedNode = result.updatedSituationGraph.nodes.find(
(node) => node.id === result.selectedQuestion.nodeId,
);
const reformulated = formulateQuestion({
node: selectedNode,
graph: result.updatedSituationGraph,
context: {
resolvedValues: ["Resolved context answer"],
},
});
expect(reformulated.question).toBe(result.selectedQuestion.question);
expect(clone(result.updatedSituationGraph)).toEqual(
result.updatedSituationGraph,
);
});
}
it("reports all five selected unknowns and strategies", () => {
const summary = questionPriorityGeneralisationFixtures.map((fixture) => {
const result = applyValidatedProposal({
situationGraph: fixture.graph,
proposal: buildResolutionProposal(fixture.graph),
});
expect(result.success).toBe(true);
return {
scenario: fixture.scenario,
nodeId: result.selectedQuestion.nodeId,
strategy: result.selectedQuestion.strategy,
};
});
expect(summary).toMatchInlineSnapshot(`
[
{
"nodeId": "hire-success-criteria",
"scenario": "Should we hire another engineer?",
"strategy": "decision criterion",
},
{
"nodeId": "van-reliability-threshold",
"scenario": "Should we replace the delivery vans?",
"strategy": "decision criterion",
},
{
"nodeId": "country-value-threshold",
"scenario": "Should we launch in another country?",
"strategy": "actor/customer",
},
{
"nodeId": "project-benefit-threshold",
"scenario": "Should we continue a project that is over budget?",
"strategy": "decision criterion",
},
{
"nodeId": "support-value-threshold",
"scenario": "Should we introduce a paid support tier?",
"strategy": "actor/customer",
},
]
`);
});
});