test: generalise question priority across decisions
This commit is contained in:
@@ -0,0 +1,48 @@
|
|||||||
|
# v0.5 Question Priority Generalisation
|
||||||
|
|
||||||
|
## Hypothesis
|
||||||
|
|
||||||
|
The current deterministic unknown selector and graph-context question formulator should generalise across several decision types by selecting a foundational unknown before downstream implementation or pricing leaves.
|
||||||
|
|
||||||
|
## Scenarios
|
||||||
|
|
||||||
|
1. Should we hire another engineer?
|
||||||
|
2. Should we replace the delivery vans?
|
||||||
|
3. Should we launch in another country?
|
||||||
|
4. Should we continue a project that is over budget?
|
||||||
|
5. Should we introduce a paid support tier?
|
||||||
|
|
||||||
|
## Results
|
||||||
|
|
||||||
|
| Scenario | Selected unknown | Strategy | Pass/Fail |
|
||||||
|
| ---------------------------- | --------------------------- | -------------------- | --------- |
|
||||||
|
| Hire another engineer | `hire-success-criteria` | `decision criterion` | Pass |
|
||||||
|
| Replace the delivery vans | `van-reliability-threshold` | `decision criterion` | Pass |
|
||||||
|
| Launch in another country | `country-value-threshold` | `actor/customer` | Pass |
|
||||||
|
| Continue over-budget project | `project-benefit-threshold` | `decision criterion` | Pass |
|
||||||
|
| Introduce paid support tier | `support-value-threshold` | `actor/customer` | Pass |
|
||||||
|
|
||||||
|
## Repeated failure patterns
|
||||||
|
|
||||||
|
Two repeated structural formulation failures appeared before the final pass:
|
||||||
|
|
||||||
|
1. **Constraint language in surrounding graph context outranked node-local decision-threshold language** in more than one case.
|
||||||
|
2. **Baseline language in surrounding graph context outranked node-local threshold language** in more than one case.
|
||||||
|
|
||||||
|
Both failures affected formulation strategy, not deterministic unknown selection.
|
||||||
|
|
||||||
|
## Code change made
|
||||||
|
|
||||||
|
A small deterministic change was made in `lib/graph/question-formulator.js`:
|
||||||
|
|
||||||
|
- prefer node-local `definition` language before broader criterion inference
|
||||||
|
- prefer node-local `decision criterion` language before context-only `constraint` inference
|
||||||
|
- only treat `baseline` or `constraint` as primary when the selected node itself carries that language, otherwise allow them as fallback strategies later
|
||||||
|
|
||||||
|
No architecture, UI, persistence, prompt, scoring, additional model turns, or provider calls were added.
|
||||||
|
|
||||||
|
## Remaining limitations
|
||||||
|
|
||||||
|
- In two passing cases, the selector chose a threshold-style foundational node while the formulator still used an `actor/customer` strategy because related context strongly referenced customers or recipients.
|
||||||
|
- This experiment is fixture-driven and deterministic; it is useful for regression protection, not scientific validation.
|
||||||
|
- The suite exercises the production path without model calls, but it does not prove behaviour over arbitrary real-world graph structures.
|
||||||
@@ -141,6 +141,9 @@ function toGerundPhrase(phrase) {
|
|||||||
|
|
||||||
function detectStrategy({ node, graph, relatedNodes, combinedText, meaning }) {
|
function detectStrategy({ node, graph, relatedNodes, combinedText, meaning }) {
|
||||||
const text = normaliseText(combinedText);
|
const text = normaliseText(combinedText);
|
||||||
|
const nodeText = normaliseText(
|
||||||
|
`${node?.label || ""} ${node?.description || ""}`,
|
||||||
|
);
|
||||||
const relatedText = normaliseText(
|
const relatedText = normaliseText(
|
||||||
relatedNodes
|
relatedNodes
|
||||||
.map((relatedNode) => `${relatedNode.label} ${relatedNode.description}`)
|
.map((relatedNode) => `${relatedNode.label} ${relatedNode.description}`)
|
||||||
@@ -160,19 +163,25 @@ function detectStrategy({ node, graph, relatedNodes, combinedText, meaning }) {
|
|||||||
`${text} ${relatedText} ${resolvedValues.join(" ")}`,
|
`${text} ${relatedText} ${resolvedValues.join(" ")}`,
|
||||||
) || Boolean(actionPhrase);
|
) || Boolean(actionPhrase);
|
||||||
|
|
||||||
if (
|
const hasConstraintLanguage =
|
||||||
/\b(constraint|limit|budget|deadline|requirement|regulation|capacity)\b/.test(
|
/\b(constraint|limit|budget|deadline|requirement|regulation|capacity)\b/.test(
|
||||||
text,
|
text,
|
||||||
)
|
);
|
||||||
) {
|
const hasPrimaryConstraintLanguage =
|
||||||
return { strategy: "constraint", meaning, actionPhrase };
|
/\b(constraint|limit|budget|deadline|requirement|regulation|capacity)\b/.test(
|
||||||
}
|
nodeText,
|
||||||
|
);
|
||||||
|
|
||||||
if (/\b(customer|user|buyer|stakeholder|recipient|audience)\b/.test(text)) {
|
if (/\b(customer|user|buyer|stakeholder|recipient|audience)\b/.test(text)) {
|
||||||
return { strategy: "actor/customer", meaning, actionPhrase };
|
return { strategy: "actor/customer", meaning, actionPhrase };
|
||||||
}
|
}
|
||||||
|
|
||||||
if (/\b(before|previous|baseline|prior|comparable state)\b/.test(text)) {
|
const hasBaselineLanguage =
|
||||||
|
/\b(before|previous|baseline|prior|comparable state)\b/.test(text);
|
||||||
|
const hasPrimaryBaselineLanguage =
|
||||||
|
/\b(before|previous|baseline|prior|comparable state)\b/.test(nodeText);
|
||||||
|
|
||||||
|
if (hasBaselineLanguage && hasPrimaryBaselineLanguage) {
|
||||||
return { strategy: "baseline", meaning, actionPhrase };
|
return { strategy: "baseline", meaning, actionPhrase };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -182,6 +191,12 @@ function detectStrategy({ node, graph, relatedNodes, combinedText, meaning }) {
|
|||||||
|
|
||||||
const hasDefinitionLanguage =
|
const hasDefinitionLanguage =
|
||||||
/\b(define|definition|meaning|term|terminology)\b/.test(text);
|
/\b(define|definition|meaning|term|terminology)\b/.test(text);
|
||||||
|
const hasPrimaryDefinitionLanguage =
|
||||||
|
/\b(define|definition|meaning|term|terminology)\b/.test(nodeText);
|
||||||
|
const hasCriteriaLanguage =
|
||||||
|
/\b(success criteria|success threshold|threshold|decision criteria|criterion|justify|sufficient)\b/.test(
|
||||||
|
nodeText,
|
||||||
|
);
|
||||||
const hasDecisionValueLanguage =
|
const hasDecisionValueLanguage =
|
||||||
decisionContext &&
|
decisionContext &&
|
||||||
/\b(value|commercial value|commercial viability|viability|justify|sufficient|success|threshold|criterion)\b/.test(
|
/\b(value|commercial value|commercial viability|viability|justify|sufficient|success|threshold|criterion)\b/.test(
|
||||||
@@ -196,14 +211,30 @@ function detectStrategy({ node, graph, relatedNodes, combinedText, meaning }) {
|
|||||||
return { strategy: "measurement", meaning, actionPhrase };
|
return { strategy: "measurement", meaning, actionPhrase };
|
||||||
}
|
}
|
||||||
|
|
||||||
if (/\b(define|definition|meaning|term|terminology)\b/.test(text)) {
|
if (hasPrimaryDefinitionLanguage) {
|
||||||
return { strategy: "definition", meaning, actionPhrase };
|
return { strategy: "definition", meaning, actionPhrase };
|
||||||
}
|
}
|
||||||
|
|
||||||
if (hasDecisionValueLanguage) {
|
if (hasDecisionValueLanguage || hasCriteriaLanguage) {
|
||||||
return { strategy: "decision criterion", meaning, actionPhrase };
|
return { strategy: "decision criterion", meaning, actionPhrase };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (hasConstraintLanguage && hasPrimaryConstraintLanguage) {
|
||||||
|
return { strategy: "constraint", meaning, actionPhrase };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (hasDefinitionLanguage) {
|
||||||
|
return { strategy: "definition", meaning, actionPhrase };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (hasBaselineLanguage) {
|
||||||
|
return { strategy: "baseline", meaning, actionPhrase };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (hasConstraintLanguage) {
|
||||||
|
return { strategy: "constraint", meaning, actionPhrase };
|
||||||
|
}
|
||||||
|
|
||||||
if (/\b(evidence|proof|validate|validation|signal|demand)\b/.test(text)) {
|
if (/\b(evidence|proof|validate|validation|signal|demand)\b/.test(text)) {
|
||||||
return { strategy: "evidence", meaning, actionPhrase };
|
return { strategy: "evidence", meaning, actionPhrase };
|
||||||
}
|
}
|
||||||
|
|||||||
+433
@@ -0,0 +1,433 @@
|
|||||||
|
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||||
|
|
||||||
|
function makeScenarioGraph({
|
||||||
|
scenario,
|
||||||
|
decisionNode,
|
||||||
|
answeredContextUnknown,
|
||||||
|
foundationalUnknown,
|
||||||
|
consequentialUnknown,
|
||||||
|
downstreamLeaf,
|
||||||
|
}) {
|
||||||
|
const nodes = [
|
||||||
|
decisionNode,
|
||||||
|
answeredContextUnknown,
|
||||||
|
foundationalUnknown,
|
||||||
|
consequentialUnknown,
|
||||||
|
downstreamLeaf,
|
||||||
|
];
|
||||||
|
|
||||||
|
const edges = [
|
||||||
|
makeEdge({
|
||||||
|
id: `${decisionNode.id}-to-${foundationalUnknown.id}`,
|
||||||
|
fromNodeId: decisionNode.id,
|
||||||
|
toNodeId: foundationalUnknown.id,
|
||||||
|
relationship: "depends_on",
|
||||||
|
description: `${decisionNode.label} depends on ${foundationalUnknown.label}.`,
|
||||||
|
}),
|
||||||
|
makeEdge({
|
||||||
|
id: `${answeredContextUnknown.id}-to-${consequentialUnknown.id}`,
|
||||||
|
fromNodeId: answeredContextUnknown.id,
|
||||||
|
toNodeId: consequentialUnknown.id,
|
||||||
|
relationship: "depends_on",
|
||||||
|
description: `${consequentialUnknown.label} was surfaced from resolved context.`,
|
||||||
|
}),
|
||||||
|
makeEdge({
|
||||||
|
id: `${foundationalUnknown.id}-to-${consequentialUnknown.id}`,
|
||||||
|
fromNodeId: foundationalUnknown.id,
|
||||||
|
toNodeId: consequentialUnknown.id,
|
||||||
|
relationship: "depends_on",
|
||||||
|
description: `${consequentialUnknown.label} depends on ${foundationalUnknown.label}.`,
|
||||||
|
}),
|
||||||
|
makeEdge({
|
||||||
|
id: `${consequentialUnknown.id}-to-${downstreamLeaf.id}`,
|
||||||
|
fromNodeId: consequentialUnknown.id,
|
||||||
|
toNodeId: downstreamLeaf.id,
|
||||||
|
relationship: "depends_on",
|
||||||
|
description: `${downstreamLeaf.label} depends on ${consequentialUnknown.label}.`,
|
||||||
|
}),
|
||||||
|
];
|
||||||
|
|
||||||
|
return makeGraph({
|
||||||
|
centralStatement: scenario,
|
||||||
|
nodes,
|
||||||
|
edges,
|
||||||
|
activeUnknownNodeId: answeredContextUnknown.id,
|
||||||
|
resolvedNodeIds: [],
|
||||||
|
currentSummary: "Generalisation fixture graph",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export const questionPriorityGeneralisationFixtures = [
|
||||||
|
{
|
||||||
|
key: "hire-engineer",
|
||||||
|
scenario: "Should we hire another engineer?",
|
||||||
|
decisionType: "resourcing decision",
|
||||||
|
acceptableFoundationalUnknownNodeIds: [
|
||||||
|
"hire-success-criteria",
|
||||||
|
"hire-bottleneck",
|
||||||
|
],
|
||||||
|
prohibitedFirstTopics: ["salary", "job advert", "programming language"],
|
||||||
|
acceptableQuestionStrategies: ["decision criterion", "constraint"],
|
||||||
|
notes:
|
||||||
|
"The first question should establish whether more engineering capacity is justified before compensation or implementation details.",
|
||||||
|
graph: makeScenarioGraph({
|
||||||
|
scenario: "Should we hire another engineer?",
|
||||||
|
decisionNode: makeNode({
|
||||||
|
id: "hire-decision",
|
||||||
|
label: "Hiring another engineer decision",
|
||||||
|
description: "Decision about increasing engineering capacity.",
|
||||||
|
kind: "state",
|
||||||
|
status: "known",
|
||||||
|
confidence: "medium",
|
||||||
|
value: "Deciding whether to hire another engineer",
|
||||||
|
childIds: ["hire-success-criteria"],
|
||||||
|
}),
|
||||||
|
answeredContextUnknown: makeNode({
|
||||||
|
id: "hire-delays-known",
|
||||||
|
label: "Delivery delays established",
|
||||||
|
description:
|
||||||
|
"Need to confirm whether recent delivery delays are real because this context determines whether a capacity decision is even relevant.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "high",
|
||||||
|
value:
|
||||||
|
"The roadmap is slipping because the current team cannot clear the queue.",
|
||||||
|
childIds: ["hire-bottleneck"],
|
||||||
|
}),
|
||||||
|
foundationalUnknown: makeNode({
|
||||||
|
id: "hire-success-criteria",
|
||||||
|
label: "Hiring success threshold",
|
||||||
|
description:
|
||||||
|
"Need the success threshold because the hiring decision depends on what improvement would justify adding headcount.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "high",
|
||||||
|
parentId: "hire-decision",
|
||||||
|
childIds: ["hire-bottleneck"],
|
||||||
|
}),
|
||||||
|
consequentialUnknown: makeNode({
|
||||||
|
id: "hire-bottleneck",
|
||||||
|
label: "Primary delivery bottleneck",
|
||||||
|
description:
|
||||||
|
"Need the main bottleneck because the team must know whether another engineer would relieve the limiting constraint.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "high",
|
||||||
|
dependsOn: ["hire-success-criteria"],
|
||||||
|
parentId: "hire-success-criteria",
|
||||||
|
childIds: ["hire-salary"],
|
||||||
|
}),
|
||||||
|
downstreamLeaf: makeNode({
|
||||||
|
id: "hire-salary",
|
||||||
|
label: "Engineer salary budget",
|
||||||
|
description:
|
||||||
|
"Need the salary range because compensation planning comes after the hiring case is established.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
dependsOn: ["hire-bottleneck"],
|
||||||
|
parentId: "hire-bottleneck",
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: "replace-vans",
|
||||||
|
scenario: "Should we replace the delivery vans?",
|
||||||
|
decisionType: "asset replacement decision",
|
||||||
|
acceptableFoundationalUnknownNodeIds: [
|
||||||
|
"van-reliability-threshold",
|
||||||
|
"van-service-constraint",
|
||||||
|
],
|
||||||
|
prohibitedFirstTopics: [
|
||||||
|
"purchase price",
|
||||||
|
"paint colour",
|
||||||
|
"finance provider",
|
||||||
|
],
|
||||||
|
acceptableQuestionStrategies: ["decision criterion", "constraint"],
|
||||||
|
notes:
|
||||||
|
"The first question should establish whether the fleet is failing a threshold that justifies replacement.",
|
||||||
|
graph: makeScenarioGraph({
|
||||||
|
scenario: "Should we replace the delivery vans?",
|
||||||
|
decisionNode: makeNode({
|
||||||
|
id: "van-decision",
|
||||||
|
label: "Replace delivery vans decision",
|
||||||
|
description: "Decision about replacing the current delivery fleet.",
|
||||||
|
kind: "state",
|
||||||
|
status: "known",
|
||||||
|
confidence: "medium",
|
||||||
|
value: "Deciding whether to replace the delivery vans",
|
||||||
|
childIds: ["van-reliability-threshold"],
|
||||||
|
}),
|
||||||
|
answeredContextUnknown: makeNode({
|
||||||
|
id: "van-breakdowns-known",
|
||||||
|
label: "Breakdown trend confirmed",
|
||||||
|
description:
|
||||||
|
"Need to confirm whether the recent rise in breakdowns is real because that context determines whether fleet replacement is relevant.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "high",
|
||||||
|
value:
|
||||||
|
"Breakdowns and missed deliveries have increased over the last quarter.",
|
||||||
|
childIds: ["van-service-constraint"],
|
||||||
|
}),
|
||||||
|
foundationalUnknown: makeNode({
|
||||||
|
id: "van-reliability-threshold",
|
||||||
|
label: "Replacement justification threshold",
|
||||||
|
description:
|
||||||
|
"Need the threshold because the replacement decision depends on what level of reliability loss is enough to justify replacing the fleet.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "high",
|
||||||
|
parentId: "van-decision",
|
||||||
|
childIds: ["van-service-constraint"],
|
||||||
|
}),
|
||||||
|
consequentialUnknown: makeNode({
|
||||||
|
id: "van-service-constraint",
|
||||||
|
label: "Operational service constraint",
|
||||||
|
description:
|
||||||
|
"Need the limiting service constraint because the team must know how vehicle unreliability is affecting deliveries before comparing purchasing options.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "high",
|
||||||
|
dependsOn: ["van-reliability-threshold"],
|
||||||
|
parentId: "van-reliability-threshold",
|
||||||
|
childIds: ["van-price"],
|
||||||
|
}),
|
||||||
|
downstreamLeaf: makeNode({
|
||||||
|
id: "van-price",
|
||||||
|
label: "Exact replacement purchase price",
|
||||||
|
description:
|
||||||
|
"Need the exact purchase price because financing analysis comes after replacement is justified.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
dependsOn: ["van-service-constraint"],
|
||||||
|
parentId: "van-service-constraint",
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: "launch-country",
|
||||||
|
scenario: "Should we launch in another country?",
|
||||||
|
decisionType: "market expansion decision",
|
||||||
|
acceptableFoundationalUnknownNodeIds: [
|
||||||
|
"country-customer",
|
||||||
|
"country-value-threshold",
|
||||||
|
],
|
||||||
|
prohibitedFirstTopics: [
|
||||||
|
"launch date",
|
||||||
|
"office location",
|
||||||
|
"advertising channel",
|
||||||
|
],
|
||||||
|
acceptableQuestionStrategies: ["actor/customer", "decision criterion"],
|
||||||
|
notes:
|
||||||
|
"The first question should clarify the customer or value case for expansion before rollout logistics.",
|
||||||
|
graph: makeScenarioGraph({
|
||||||
|
scenario: "Should we launch in another country?",
|
||||||
|
decisionNode: makeNode({
|
||||||
|
id: "country-decision",
|
||||||
|
label: "Launch in another country decision",
|
||||||
|
description: "Decision about entering a new national market.",
|
||||||
|
kind: "state",
|
||||||
|
status: "known",
|
||||||
|
confidence: "medium",
|
||||||
|
value: "Deciding whether to launch in another country",
|
||||||
|
childIds: ["country-customer"],
|
||||||
|
}),
|
||||||
|
answeredContextUnknown: makeNode({
|
||||||
|
id: "country-interest-known",
|
||||||
|
label: "Inbound interest confirmed",
|
||||||
|
description:
|
||||||
|
"Need to confirm whether inbound interest from another country is real because that context determines whether expansion is relevant.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
value:
|
||||||
|
"Prospective customers from another country are asking for access.",
|
||||||
|
childIds: ["country-value-threshold"],
|
||||||
|
}),
|
||||||
|
foundationalUnknown: makeNode({
|
||||||
|
id: "country-customer",
|
||||||
|
label: "Relevant customer in the new country",
|
||||||
|
description:
|
||||||
|
"Need the relevant customer because the expansion decision depends on who experiences the problem or receives the value in that market.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "high",
|
||||||
|
parentId: "country-decision",
|
||||||
|
childIds: ["country-value-threshold"],
|
||||||
|
}),
|
||||||
|
consequentialUnknown: makeNode({
|
||||||
|
id: "country-value-threshold",
|
||||||
|
label: "Expansion value threshold",
|
||||||
|
description:
|
||||||
|
"Need the value threshold because the team must know what evidence of demand or value would justify entering the new country.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "high",
|
||||||
|
dependsOn: ["country-customer"],
|
||||||
|
parentId: "country-customer",
|
||||||
|
childIds: ["country-launch-date"],
|
||||||
|
}),
|
||||||
|
downstreamLeaf: makeNode({
|
||||||
|
id: "country-launch-date",
|
||||||
|
label: "Country launch date",
|
||||||
|
description:
|
||||||
|
"Need the launch date because rollout planning follows once the expansion case is established.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
dependsOn: ["country-value-threshold"],
|
||||||
|
parentId: "country-value-threshold",
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: "over-budget-project",
|
||||||
|
scenario: "Should we continue a project that is over budget?",
|
||||||
|
decisionType: "continuation decision",
|
||||||
|
acceptableFoundationalUnknownNodeIds: [
|
||||||
|
"project-benefit-threshold",
|
||||||
|
"project-remaining-benefit",
|
||||||
|
],
|
||||||
|
prohibitedFirstTopics: ["sunk cost", "project logo", "final launch date"],
|
||||||
|
acceptableQuestionStrategies: ["decision criterion", "objective"],
|
||||||
|
notes:
|
||||||
|
"The first question should establish remaining value or success threshold before sunk-cost framing or launch timing.",
|
||||||
|
graph: makeScenarioGraph({
|
||||||
|
scenario: "Should we continue a project that is over budget?",
|
||||||
|
decisionNode: makeNode({
|
||||||
|
id: "project-decision",
|
||||||
|
label: "Continue over-budget project decision",
|
||||||
|
description:
|
||||||
|
"Decision about continuing a project that has exceeded budget.",
|
||||||
|
kind: "state",
|
||||||
|
status: "known",
|
||||||
|
confidence: "medium",
|
||||||
|
value: "Deciding whether to continue the over-budget project",
|
||||||
|
childIds: ["project-benefit-threshold"],
|
||||||
|
}),
|
||||||
|
answeredContextUnknown: makeNode({
|
||||||
|
id: "project-overrun-known",
|
||||||
|
label: "Budget overrun confirmed",
|
||||||
|
description:
|
||||||
|
"Need to confirm whether the project is materially over budget because that context determines whether a continuation decision is relevant.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "high",
|
||||||
|
value: "The project has exceeded its approved budget by 35 percent.",
|
||||||
|
childIds: ["project-remaining-benefit"],
|
||||||
|
}),
|
||||||
|
foundationalUnknown: makeNode({
|
||||||
|
id: "project-benefit-threshold",
|
||||||
|
label: "Continuation success threshold",
|
||||||
|
description:
|
||||||
|
"Need the threshold because the continuation decision depends on what remaining benefit would still justify completing the project.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "high",
|
||||||
|
parentId: "project-decision",
|
||||||
|
childIds: ["project-remaining-benefit"],
|
||||||
|
}),
|
||||||
|
consequentialUnknown: makeNode({
|
||||||
|
id: "project-remaining-benefit",
|
||||||
|
label: "Remaining project benefit",
|
||||||
|
description:
|
||||||
|
"Need the remaining benefit because the team must know what value is still achievable before deciding whether to continue.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "high",
|
||||||
|
dependsOn: ["project-benefit-threshold"],
|
||||||
|
parentId: "project-benefit-threshold",
|
||||||
|
childIds: ["project-launch-date"],
|
||||||
|
}),
|
||||||
|
downstreamLeaf: makeNode({
|
||||||
|
id: "project-launch-date",
|
||||||
|
label: "Final launch date",
|
||||||
|
description:
|
||||||
|
"Need the final launch date because scheduling details only matter after remaining value is established.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
dependsOn: ["project-remaining-benefit"],
|
||||||
|
parentId: "project-remaining-benefit",
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: "paid-support-tier",
|
||||||
|
scenario: "Should we introduce a paid support tier?",
|
||||||
|
decisionType: "commercial packaging decision",
|
||||||
|
acceptableFoundationalUnknownNodeIds: [
|
||||||
|
"support-customer",
|
||||||
|
"support-value-threshold",
|
||||||
|
],
|
||||||
|
prohibitedFirstTopics: [
|
||||||
|
"subscription price",
|
||||||
|
"payment provider",
|
||||||
|
"tier name",
|
||||||
|
],
|
||||||
|
acceptableQuestionStrategies: ["actor/customer", "decision criterion"],
|
||||||
|
notes:
|
||||||
|
"The first question should establish who values paid support or what outcome would justify offering it before pricing details.",
|
||||||
|
graph: makeScenarioGraph({
|
||||||
|
scenario: "Should we introduce a paid support tier?",
|
||||||
|
decisionNode: makeNode({
|
||||||
|
id: "support-decision",
|
||||||
|
label: "Introduce paid support tier decision",
|
||||||
|
description: "Decision about adding a paid support offering.",
|
||||||
|
kind: "state",
|
||||||
|
status: "known",
|
||||||
|
confidence: "medium",
|
||||||
|
value: "Deciding whether to introduce a paid support tier",
|
||||||
|
childIds: ["support-customer"],
|
||||||
|
}),
|
||||||
|
answeredContextUnknown: makeNode({
|
||||||
|
id: "support-requests-known",
|
||||||
|
label: "Support request pattern confirmed",
|
||||||
|
description:
|
||||||
|
"Need to confirm whether repeated requests for faster support responses are real because that context determines whether a paid tier is relevant.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "high",
|
||||||
|
value:
|
||||||
|
"Some users are asking for guaranteed response times and escalation help.",
|
||||||
|
childIds: ["support-value-threshold"],
|
||||||
|
}),
|
||||||
|
foundationalUnknown: makeNode({
|
||||||
|
id: "support-customer",
|
||||||
|
label: "Customer willing to pay for support",
|
||||||
|
description:
|
||||||
|
"Need the customer because the decision depends on who experiences enough support pain or receives enough value to pay for a support tier.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "high",
|
||||||
|
parentId: "support-decision",
|
||||||
|
childIds: ["support-value-threshold"],
|
||||||
|
}),
|
||||||
|
consequentialUnknown: makeNode({
|
||||||
|
id: "support-value-threshold",
|
||||||
|
label: "Paid support value threshold",
|
||||||
|
description:
|
||||||
|
"Need the value threshold because the team must know what outcome would justify introducing paid support before setting packaging details.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "high",
|
||||||
|
dependsOn: ["support-customer"],
|
||||||
|
parentId: "support-customer",
|
||||||
|
childIds: ["support-price"],
|
||||||
|
}),
|
||||||
|
downstreamLeaf: makeNode({
|
||||||
|
id: "support-price",
|
||||||
|
label: "Support subscription price",
|
||||||
|
description:
|
||||||
|
"Need the subscription price because pricing and payment setup come after the support value case is established.",
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
dependsOn: ["support-value-threshold"],
|
||||||
|
parentId: "support-value-threshold",
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
];
|
||||||
@@ -0,0 +1,155 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { applyValidatedProposal } from "@/lib/graph/apply-proposal.js";
|
||||||
|
import { formulateQuestion } from "@/lib/graph/question-formulator.js";
|
||||||
|
import { selectActiveUnknownCandidate } from "@/lib/graph/utils.js";
|
||||||
|
import { questionPriorityGeneralisationFixtures } from "@/tests/fixtures/question-priority-generalisation.js";
|
||||||
|
|
||||||
|
function clone(value) {
|
||||||
|
return JSON.parse(JSON.stringify(value));
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildResolutionProposal(graph) {
|
||||||
|
const activeNode = graph.nodes.find(
|
||||||
|
(node) => node.id === graph.activeUnknownNodeId,
|
||||||
|
);
|
||||||
|
const placeholderCandidate = graph.nodes.find(
|
||||||
|
(node) => node.kind === "unknown" && node.id !== activeNode.id,
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
addedNodes: [],
|
||||||
|
updatedNodes: [
|
||||||
|
{
|
||||||
|
nodeId: activeNode.id,
|
||||||
|
previousStatus: activeNode.status,
|
||||||
|
newStatus: "resolved",
|
||||||
|
previousValue: activeNode.value ?? null,
|
||||||
|
newValue: activeNode.value ?? "Resolved context answer",
|
||||||
|
reason:
|
||||||
|
"The resolved context unknown is treated as answered for fixture progression.",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
addedEdges: [],
|
||||||
|
removedEdgeIds: [],
|
||||||
|
resolvedUnknownNodeIds: [activeNode.id],
|
||||||
|
affectedNodeIds: [],
|
||||||
|
selectedQuestion: {
|
||||||
|
nodeId: placeholderCandidate?.id,
|
||||||
|
question: "Placeholder candidate question?",
|
||||||
|
reason: "Candidate only; deterministic selector should override it.",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function assertQuestionStructure(question) {
|
||||||
|
expect(question.match(/\?/g) || []).toHaveLength(1);
|
||||||
|
expect(question).not.toMatch(/\?\s*(and|or)\b/i);
|
||||||
|
expect(question).not.toMatch(/^What is\s+/i);
|
||||||
|
expect(question).toMatch(/^(What|Who|When)\b/);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("question priority generalisation", () => {
|
||||||
|
for (const fixture of questionPriorityGeneralisationFixtures) {
|
||||||
|
it(`${fixture.scenario} selects a foundational unknown and singular answerable strategy`, () => {
|
||||||
|
const originalGraph = clone(fixture.graph);
|
||||||
|
const deterministicSelection = selectActiveUnknownCandidate(
|
||||||
|
fixture.graph,
|
||||||
|
[fixture.graph.activeUnknownNodeId],
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = applyValidatedProposal({
|
||||||
|
situationGraph: fixture.graph,
|
||||||
|
proposal: buildResolutionProposal(fixture.graph),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
expect(fixture.graph).toEqual(originalGraph);
|
||||||
|
expect(result.graphUpdate.selectedQuestion?.question).toBe(
|
||||||
|
"Placeholder candidate question?",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(deterministicSelection.nodeId).toBe(
|
||||||
|
result.selectedQuestion.nodeId,
|
||||||
|
);
|
||||||
|
expect(fixture.acceptableFoundationalUnknownNodeIds).toContain(
|
||||||
|
result.selectedQuestion.nodeId,
|
||||||
|
);
|
||||||
|
expect(result.selectedQuestion.nodeId).not.toBe(
|
||||||
|
fixture.graph.nodes[fixture.graph.nodes.length - 1].id,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(fixture.acceptableQuestionStrategies).toContain(
|
||||||
|
result.selectedQuestion.strategy,
|
||||||
|
);
|
||||||
|
assertQuestionStructure(result.selectedQuestion.question);
|
||||||
|
|
||||||
|
const lowerQuestion = result.selectedQuestion.question.toLowerCase();
|
||||||
|
for (const topic of fixture.prohibitedFirstTopics) {
|
||||||
|
expect(lowerQuestion).not.toContain(topic.toLowerCase());
|
||||||
|
}
|
||||||
|
|
||||||
|
const selectedNode = result.updatedSituationGraph.nodes.find(
|
||||||
|
(node) => node.id === result.selectedQuestion.nodeId,
|
||||||
|
);
|
||||||
|
const reformulated = formulateQuestion({
|
||||||
|
node: selectedNode,
|
||||||
|
graph: result.updatedSituationGraph,
|
||||||
|
context: {
|
||||||
|
resolvedValues: ["Resolved context answer"],
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(reformulated.question).toBe(result.selectedQuestion.question);
|
||||||
|
expect(clone(result.updatedSituationGraph)).toEqual(
|
||||||
|
result.updatedSituationGraph,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
it("reports all five selected unknowns and strategies", () => {
|
||||||
|
const summary = questionPriorityGeneralisationFixtures.map((fixture) => {
|
||||||
|
const result = applyValidatedProposal({
|
||||||
|
situationGraph: fixture.graph,
|
||||||
|
proposal: buildResolutionProposal(fixture.graph),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
|
||||||
|
return {
|
||||||
|
scenario: fixture.scenario,
|
||||||
|
nodeId: result.selectedQuestion.nodeId,
|
||||||
|
strategy: result.selectedQuestion.strategy,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(summary).toMatchInlineSnapshot(`
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"nodeId": "hire-success-criteria",
|
||||||
|
"scenario": "Should we hire another engineer?",
|
||||||
|
"strategy": "decision criterion",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"nodeId": "van-reliability-threshold",
|
||||||
|
"scenario": "Should we replace the delivery vans?",
|
||||||
|
"strategy": "decision criterion",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"nodeId": "country-value-threshold",
|
||||||
|
"scenario": "Should we launch in another country?",
|
||||||
|
"strategy": "actor/customer",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"nodeId": "project-benefit-threshold",
|
||||||
|
"scenario": "Should we continue a project that is over budget?",
|
||||||
|
"strategy": "decision criterion",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"nodeId": "support-value-threshold",
|
||||||
|
"scenario": "Should we introduce a paid support tier?",
|
||||||
|
"strategy": "actor/customer",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
`);
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user