robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented Investigation Strategy

Browse Source
This commit is contained in:
robbond
2026-08-02 15:10:49 +01:00
parent a79a7bd524
commit 5ef9710293
7 changed files with 293 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
components/diagnostics-view.jsx
+8
View File
@@ -80,6 +80,14 @@ export default function DiagnosticsView({ result }) {
? `${validationIcons.valid} valid`
: `${validationIcons.invalid} invalid`,
},
{
label: "Investigation strategy",
value:
diagnostics.investigationStrategy?.key ||
diagnostics.investigationStrategy ||
result.selectedQuestion?.strategy ||
"?",
},
];
const errors = [
lib/graph/apply-proposal.js
+1
View File
@@ -656,6 +656,7 @@ export function applyValidatedProposal({ situationGraph, proposal }) {
formulatedQuestion?.question || deterministicSelection.question,
reason: formulatedQuestion?.reason || deterministicSelection.reason,
strategy: formulatedQuestion?.strategy,
investigationStrategy: formulatedQuestion?.investigationStrategy,
}
: null;
lib/graph/orchestrator.js
+8
View File
@@ -53,6 +53,7 @@ function buildUpdateDiagnostics({
normalisationsApplied,
graph,
graphReferenceValidation,
selectedQuestion,
}) {
return {
promptVersion: promptVersion ?? "v0.4",
@@ -66,6 +67,10 @@ function buildUpdateDiagnostics({
errors: [],
},
normalisationsApplied: normalisationsApplied ?? [],
investigationStrategy:
selectedQuestion?.investigationStrategy ??
selectedQuestion?.strategy ??
null,
};
}
@@ -284,6 +289,7 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
normalisationsApplied: parsedProposal.normalisationsApplied,
graph: situationGraph,
graphReferenceValidation: graphReferenceValidation,
selectedQuestion: null,
}),
},
statusCode:
@@ -313,6 +319,7 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
normalisationsApplied: parsedProposal.normalisationsApplied,
graph: applicationResult.updatedSituationGraph,
graphReferenceValidation: applicationResult.graphReferenceValidation,
selectedQuestion: applicationResult.selectedQuestion,
}),
};
}
@@ -328,6 +335,7 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
normalisationsApplied: parsedProposal.normalisationsApplied,
graph: situationGraph,
graphReferenceValidation,
selectedQuestion: null,
}),
};
}
lib/graph/question-formulator.js
+146 -93
View File
@@ -139,7 +139,41 @@ function toGerundPhrase(phrase) {
return [gerund, ...rest].join(" ");
}
function detectStrategy({ node, graph, relatedNodes, combinedText, meaning }) {
function buildInvestigationStrategy({
key,
reason,
node,
graph,
relatedNodes,
meaning,
actionPhrase,
}) {
return {
key,
reason,
nodeId: node?.id ?? null,
nodeLabel: node?.label ?? null,
meaning,
actionPhrase,
relatedNodeIds: relatedNodes.map((relatedNode) => relatedNode.id),
centralStatement: graph?.centralStatement ?? null,
};
}
export function selectInvestigationStrategy({ node, graph, context = {} }) {
const relatedNodes = collectRelatedNodes(node, graph);
const meaning = extractMeaning(node);
const combinedText = [
node?.label,
node?.description,
...relatedNodes.map((relatedNode) => relatedNode.label),
...relatedNodes.map((relatedNode) => relatedNode.description),
graph?.centralStatement,
...(context.resolvedValues || []),
]
.filter(Boolean)
.join(" ");
const text = normaliseText(combinedText);
const nodeText = normaliseText(
`${node?.label || ""} ${node?.description || ""}`,
@@ -172,21 +206,22 @@ function detectStrategy({ node, graph, relatedNodes, combinedText, meaning }) {
nodeText,
);
if (/\b(customer|user|buyer|stakeholder|recipient|audience)\b/.test(text)) {
return { strategy: "actor/customer", meaning, actionPhrase };
}
const hasBaselineLanguage =
/\b(before|previous|baseline|prior|comparable state)\b/.test(text);
const hasPrimaryBaselineLanguage =
/\b(before|previous|baseline|prior|comparable state)\b/.test(nodeText);
if (hasBaselineLanguage && hasPrimaryBaselineLanguage) {
return { strategy: "baseline", meaning, actionPhrase };
}
if (/\b(when|timing|timeline|duration|sequence|milestone)\b/.test(text)) {
return { strategy: "transition/timing", meaning, actionPhrase };
return buildInvestigationStrategy({
key: "baseline_reconstruction",
reason:
"Selected because the unknown explicitly references a missing previous or baseline state.",
node,
graph,
relatedNodes,
meaning,
actionPhrase,
});
}
const hasDefinitionLanguage =
@@ -206,84 +241,116 @@ function detectStrategy({ node, graph, relatedNodes, combinedText, meaning }) {
/\b(metric|measure|measurable|roi|revenue projection|benchmark)\b/.test(
text,
);
const hasEvidenceLanguage =
/\b(evidence|proof|validate|validation|signal|demand)\b/.test(text) ||
node?.kind === "reported_claim" ||
node?.kind === "conclusion" ||
/\b(claim|assertion|true|false)\b/.test(text);
const hasContradictionLanguage =
/\b(contradiction|contradict|conflict|inconsistent|inconsistency|disagree|mismatch)\b/.test(
`${text} ${relatedText}`,
) ||
relatedNodes.some(
(relatedNode) =>
relatedNode.status === "contradicted" ||
relatedNode.kind === "conclusion",
);
if (hasDecisionValueLanguage && hasMeasurementLanguage) {
return { strategy: "measurement", meaning, actionPhrase };
}
if (hasPrimaryDefinitionLanguage) {
return { strategy: "definition", meaning, actionPhrase };
if (hasPrimaryDefinitionLanguage || hasDefinitionLanguage) {
return buildInvestigationStrategy({
key: "definition",
reason:
"Selected because the unknown is primarily about clarifying what a term means in this case.",
node,
graph,
relatedNodes,
meaning,
actionPhrase,
});
}
if (hasDecisionValueLanguage || hasCriteriaLanguage) {
return { strategy: "decision criterion", meaning, actionPhrase };
return buildInvestigationStrategy({
key: "decision_threshold",
reason:
"Selected because the unknown determines the threshold for making or justifying a decision.",
node,
graph,
relatedNodes,
meaning,
actionPhrase,
});
}
if (hasConstraintLanguage && hasPrimaryConstraintLanguage) {
return { strategy: "constraint", meaning, actionPhrase };
if (hasPrimaryBaselineLanguage || hasBaselineLanguage) {
return buildInvestigationStrategy({
key: "baseline_reconstruction",
reason:
"Selected because reconstructing the prior state is the most direct way to resolve the unknown.",
node,
graph,
relatedNodes,
meaning,
actionPhrase,
});
}
if (hasDefinitionLanguage) {
return { strategy: "definition", meaning, actionPhrase };
if (hasContradictionLanguage) {
return buildInvestigationStrategy({
key: "contradiction_resolution",
reason:
"Selected because the graph context indicates conflicting claims or inconsistent states that must be reconciled.",
node,
graph,
relatedNodes,
meaning,
actionPhrase,
});
}
if (hasBaselineLanguage) {
return { strategy: "baseline", meaning, actionPhrase };
if (hasEvidenceLanguage || hasMeasurementLanguage || hasConstraintLanguage) {
return buildInvestigationStrategy({
key: "evidence_gathering",
reason:
hasConstraintLanguage && hasPrimaryConstraintLanguage
? "Selected because evidence about the practical limiting factor is needed before the unknown can be resolved."
: "Selected because resolving the unknown requires evidence, signals, or measurable confirmation.",
node,
graph,
relatedNodes,
meaning,
actionPhrase,
});
}
if (hasConstraintLanguage) {
return { strategy: "constraint", meaning, actionPhrase };
}
if (/\b(evidence|proof|validate|validation|signal|demand)\b/.test(text)) {
return { strategy: "evidence", meaning, actionPhrase };
}
if (hasMeasurementLanguage) {
return { strategy: "measurement", meaning, actionPhrase };
}
if (
/\b(objective|goal|outcome|problem|job to be done|benefit)\b/.test(text)
) {
return { strategy: "objective", meaning, actionPhrase };
}
if (
node?.kind === "reported_claim" ||
node?.kind === "conclusion" ||
/\b(claim|assertion|true|false)\b/.test(text)
) {
return { strategy: "evidence", meaning, actionPhrase };
}
return { strategy: "generic clarification", meaning, actionPhrase };
return buildInvestigationStrategy({
key: "definition",
reason:
"Selected as the deterministic fallback because clarifying the exact meaning of the unknown is the narrowest first step.",
node,
graph,
relatedNodes,
meaning,
actionPhrase,
});
}
function buildQuestion({ strategy, meaning, actionPhrase }) {
switch (strategy) {
case "decision criterion":
return actionPhrase
? `What outcome would demonstrate enough value to justify ${toGerundPhrase(actionPhrase)}?`
function buildQuestionFromStrategy(strategy) {
switch (strategy.key) {
case "decision_threshold":
return strategy.actionPhrase
? `What outcome would demonstrate enough value to justify ${toGerundPhrase(strategy.actionPhrase)}?`
: "What outcome would be sufficient to justify this decision?";
case "definition":
return `What does ${meaning} mean in this situation?`;
case "evidence":
return `What evidence would show whether ${meaning} is true?`;
case "baseline":
return `What was the comparable state before ${meaning}?`;
case "actor/customer":
return "Who experiences the problem or receives the value in this situation?";
case "objective":
return "What outcome is this decision or effort meant to achieve?";
case "constraint":
return "What constraint most limits the available options in this situation?";
case "measurement":
return `What measure would determine whether ${meaning} is sufficient?`;
case "transition/timing":
return `When does ${meaning} become relevant in the decision or change?`;
return `What does ${strategy.meaning} mean in this situation?`;
case "evidence_gathering":
return `What evidence would show whether ${strategy.meaning} is true?`;
case "baseline_reconstruction":
return `What was the comparable state before ${strategy.meaning}?`;
case "contradiction_resolution":
return `What fact would resolve the contradiction about ${strategy.meaning}?`;
default:
return `What specific fact would resolve whether ${meaning} is true?`;
return `What specific fact would resolve whether ${strategy.meaning} is true?`;
}
}
@@ -332,36 +399,22 @@ function validateFormulatedQuestion(question, meaning) {
}
export function formulateQuestion({ node, graph, context = {} }) {
const relatedNodes = collectRelatedNodes(node, graph);
const meaning = extractMeaning(node);
const combinedText = [
node?.label,
node?.description,
...relatedNodes.map((relatedNode) => relatedNode.label),
...relatedNodes.map((relatedNode) => relatedNode.description),
graph?.centralStatement,
...(context.resolvedValues || []),
]
.filter(Boolean)
.join(" ");
const detected = detectStrategy({
const investigationStrategy = selectInvestigationStrategy({
node,
graph,
relatedNodes,
combinedText,
meaning,
context,
});
let question = buildQuestion(detected);
let question = buildQuestionFromStrategy(investigationStrategy);
if (!validateFormulatedQuestion(question, meaning)) {
question = `What evidence would resolve whether ${meaning} is true?`;
if (!validateFormulatedQuestion(question, investigationStrategy.meaning)) {
question = `What evidence would resolve whether ${investigationStrategy.meaning} is true?`;
}
return {
question,
reason: `Formulated from graph context using the ${detected.strategy} strategy.`,
strategy: detected.strategy,
reason: `Formulated from graph context using the ${investigationStrategy.key} investigation strategy.`,
strategy: investigationStrategy.key,
investigationStrategy,
};
}
tests/fixtures/question-priority-generalisation.js
Vendored
+17 -5
View File
@@ -67,7 +67,11 @@ export const questionPriorityGeneralisationFixtures = [
"hire-bottleneck",
],
prohibitedFirstTopics: ["salary", "job advert", "programming language"],
acceptableQuestionStrategies: ["decision criterion", "constraint"],
acceptableQuestionStrategies: [
"decision_threshold",
"evidence_gathering",
"definition",
],
notes:
"The first question should establish whether more engineering capacity is justified before compensation or implementation details.",
graph: makeScenarioGraph({
@@ -143,7 +147,11 @@ export const questionPriorityGeneralisationFixtures = [
"paint colour",
"finance provider",
],
acceptableQuestionStrategies: ["decision criterion", "constraint"],
acceptableQuestionStrategies: [
"decision_threshold",
"evidence_gathering",
"definition",
],
notes:
"The first question should establish whether the fleet is failing a threshold that justifies replacement.",
graph: makeScenarioGraph({
@@ -219,7 +227,7 @@ export const questionPriorityGeneralisationFixtures = [
"office location",
"advertising channel",
],
acceptableQuestionStrategies: ["actor/customer", "decision criterion"],
acceptableQuestionStrategies: ["definition", "decision_threshold"],
notes:
"The first question should clarify the customer or value case for expansion before rollout logistics.",
graph: makeScenarioGraph({
@@ -291,7 +299,7 @@ export const questionPriorityGeneralisationFixtures = [
"project-remaining-benefit",
],
prohibitedFirstTopics: ["sunk cost", "project logo", "final launch date"],
acceptableQuestionStrategies: ["decision criterion", "objective"],
acceptableQuestionStrategies: ["decision_threshold", "definition"],
notes:
"The first question should establish remaining value or success threshold before sunk-cost framing or launch timing.",
graph: makeScenarioGraph({
@@ -367,7 +375,11 @@ export const questionPriorityGeneralisationFixtures = [
"payment provider",
"tier name",
],
acceptableQuestionStrategies: ["actor/customer", "decision criterion"],
acceptableQuestionStrategies: [
"definition",
"decision_threshold",
"baseline_reconstruction",
],
notes:
"The first question should establish who values paid support or what outcome would justify offering it before pricing details.",
graph: makeScenarioGraph({
tests/graph/question-formulator.test.js
+108 -18
View File
@@ -1,5 +1,8 @@
import { describe, expect, it } from "vitest";
import { formulateQuestion } from "@/lib/graph/question-formulator.js";
import {
formulateQuestion,
selectInvestigationStrategy,
} from "@/lib/graph/question-formulator.js";
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
function makeGraphFor(node, extra = {}) {
@@ -14,7 +17,7 @@ function makeGraphFor(node, extra = {}) {
}
describe("formulateQuestion", () => {
it("commercial viability plus build decision produces a decision-criterion question", () => {
it("commercial viability plus build decision produces a decision-threshold question", () => {
const unknown = makeNode({
id: "n-commercial",
label: "Uncertainty regarding the commercial value of the product",
@@ -42,7 +45,7 @@ describe("formulateQuestion", () => {
const result = formulateQuestion({ node: unknown, graph });
expect(result.strategy).toBe("decision criterion");
expect(result.strategy).toBe("decision_threshold");
expect(result.question).toContain("What outcome");
expect(result.question.toLowerCase()).toContain("justify");
});
@@ -100,7 +103,7 @@ describe("formulateQuestion", () => {
graph: makeGraphFor(unknown),
});
expect(result.strategy).toBe("evidence");
expect(result.strategy).toBe("evidence_gathering");
expect(result.question).toContain("What evidence");
});
@@ -120,33 +123,41 @@ describe("formulateQuestion", () => {
graph: makeGraphFor(unknown),
});
expect(result.strategy).toBe("baseline");
expect(result.strategy).toBe("baseline_reconstruction");
expect(result.question).toContain("What was the comparable state before");
});
it("unknown customer produces an actor/customer question", () => {
it("conflicting claim produces a contradiction-resolution question", () => {
const unknown = makeNode({
id: "n-customer",
label: "Target customer",
id: "n-conflict",
label: "Conflicting churn claim",
description:
"Need to know the customer because value depends on who receives it.",
"Need to resolve the inconsistency because the current figures contradict each other.",
kind: "unknown",
status: "unknown",
confidence: "high",
confidence: "medium",
});
const contradiction = makeNode({
id: "n-contradiction",
label: "Contradicted report",
description: "Two sources disagree about churn.",
kind: "conclusion",
status: "contradicted",
confidence: "low",
childIds: [unknown.id],
});
const result = formulateQuestion({
node: unknown,
graph: makeGraphFor(unknown),
graph: makeGraphFor(unknown, { nodes: [contradiction] }),
});
expect(result.strategy).toBe("actor/customer");
expect(result.question).toContain(
"Who experiences the problem or receives the value",
);
expect(result.strategy).toBe("contradiction_resolution");
expect(result.question).toContain("resolve the contradiction");
});
it("constraint unknown produces a constraint question", () => {
it("constraint unknown uses evidence-gathering within the fixed strategy set", () => {
const unknown = makeNode({
id: "n-constraint",
label: "Budget constraint",
@@ -162,8 +173,87 @@ describe("formulateQuestion", () => {
graph: makeGraphFor(unknown),
});
expect(result.strategy).toBe("constraint");
expect(result.question).toContain("What constraint most limits");
expect(result.strategy).toBe("evidence_gathering");
expect(result.question).toContain("What evidence");
});
it("the same unknown can produce different questions when paired with different strategies", () => {
const unknown = makeNode({
id: "n-same-unknown",
label: "Value threshold",
description: "Need to resolve the value threshold.",
kind: "unknown",
status: "unknown",
confidence: "high",
});
const decisionGraph = makeGraphFor(unknown, {
centralStatement: "We are deciding whether to launch this product.",
nodes: [
makeNode({
id: "n-decision",
label: "Launch decision",
description: "Decision depends on the value threshold.",
kind: "state",
status: "known",
confidence: "medium",
childIds: [unknown.id],
value: "Deciding whether to launch the product",
}),
],
});
const definitionGraph = makeGraphFor(unknown, {
centralStatement:
"The team uses the term value threshold inconsistently.",
nodes: [
makeNode({
id: "n-definition",
label: "Definition disagreement",
description:
"Need a definition of value threshold before comparing options.",
kind: "state",
status: "known",
confidence: "medium",
childIds: [unknown.id],
}),
],
});
const decisionResult = formulateQuestion({
node: unknown,
graph: decisionGraph,
});
const definitionResult = formulateQuestion({
node: unknown,
graph: definitionGraph,
});
expect(decisionResult.strategy).toBe("decision_threshold");
expect(definitionResult.strategy).toBe("definition");
expect(decisionResult.question).not.toBe(definitionResult.question);
});
it("strategy selection is deterministic and explainable", () => {
const unknown = makeNode({
id: "n-threshold",
label: "Success threshold",
description:
"Need the success threshold because the decision depends on it.",
kind: "unknown",
status: "unknown",
confidence: "high",
});
const graph = makeGraphFor(unknown, {
centralStatement: "We need to decide whether to continue investing.",
});
const first = selectInvestigationStrategy({ node: unknown, graph });
const second = selectInvestigationStrategy({ node: unknown, graph });
expect(first).toEqual(second);
expect(first.key).toBe("decision_threshold");
expect(first.reason).toContain("threshold");
});
it("question is singular and answerable", () => {
tests/graph/question-priority-generalisation.test.js
+5 -5
View File
@@ -127,27 +127,27 @@ describe("question priority generalisation", () => {
{
"nodeId": "hire-success-criteria",
"scenario": "Should we hire another engineer?",
"strategy": "decision criterion",
"strategy": "decision_threshold",
},
{
"nodeId": "van-reliability-threshold",
"scenario": "Should we replace the delivery vans?",
"strategy": "decision criterion",
"strategy": "decision_threshold",
},
{
"nodeId": "country-value-threshold",
"scenario": "Should we launch in another country?",
"strategy": "actor/customer",
"strategy": "decision_threshold",
},
{
"nodeId": "project-benefit-threshold",
"scenario": "Should we continue a project that is over budget?",
"strategy": "decision criterion",
"strategy": "decision_threshold",
},
{
"nodeId": "support-value-threshold",
"scenario": "Should we introduce a paid support tier?",
"strategy": "actor/customer",
"strategy": "baseline_reconstruction",
},
]
`);