robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: make graph-backed questions authoritative

Browse Source
This commit is contained in:
robbond
2026-08-03 09:13:52 +01:00
parent ef04b9e494
commit db994d7764
4 changed files with 285 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/v0.7-question-simplicity-experiment.md
+42
View File
@@ -23,6 +23,8 @@ That violated the intended one-step reasoning discipline.
The engine should ask one question about one primary concept at a time.
**The reconstruction model may suggest a question, but only the graph-backed deterministic pipeline may select the user-facing question.**
## One-question / one-concept rule
Every user-facing question should:
@@ -85,6 +87,46 @@ This question:
- stays graph-backed
- avoids pricing or budget before problem existence is established
## Start-case authority rule
There were previously two question paths during initial analysis:
- reconstruction model `nextQuestion`
- graph-backed unknown selection and question formulation
The defect was that `startCase` copied the reconstruction `nextQuestion` directly into the normal UI.
That path is now closed.
Initial user-facing questioning now follows this pipeline:
```text
reconstruction
→ graph build
→ unresolved unknown selection
→ atomicity assessment
→ decomposition if needed
→ investigation strategy
→ question formulation
→ complexity validation
→ selectedQuestion
```
The reconstruction question is still retained in diagnostics as provenance, but it is not authoritative.
## Live result
Running the commercial-method scenario through the real environment now:
- succeeds without the enum compatibility failure
- does not show the broad reconstruction question in the UI path
- surfaces a graph-backed first question instead
- keeps the reconstruction question only in diagnostics
For the tested scenario, the user-facing first question remained:
> Who experiences this problem?
## Remaining limitations
- question-complexity assessment is still conservative and pattern-based rather than semantic in a richer linguistic sense
lib/graph/apply-proposal.js
+92
View File
@@ -5,6 +5,7 @@ import {
classifyObservationRelationship,
COMPARABILITY_REASONING_NODE_ID,
formulateQuestion,
formulateTieResolutionQuestion,
} from "./question-formulator.js";
import {
graphUpdateSchema,
@@ -1623,6 +1624,97 @@ function findNodeById(graph, nodeId) {
return (graph.nodes || []).find((node) => node.id === nodeId) || null;
}
export function determineGraphBackedQuestion({ situationGraph }) {
const graphSnapshot = cloneJsonSafe(situationGraph);
let updatedSituationGraph = cloneJsonSafe(situationGraph);
let deterministicSelection = selectActiveUnknownCandidate(
updatedSituationGraph,
updatedSituationGraph.resolvedNodeIds || [],
);
const decompositionResult = runDeterministicDecomposition({
graphSnapshot,
proposalSnapshot: {
addedNodes: [],
updatedNodes: [],
addedEdges: [],
removedEdgeIds: [],
resolvedUnknownNodeIds: [],
affectedNodeIds: [],
selectedQuestion: null,
},
updatedSituationGraph,
reasoningResolution: { reasoningStateOverride: {} },
deterministicSelection,
});
if (!decompositionResult.success) {
return decompositionResult;
}
updatedSituationGraph = decompositionResult.updatedSituationGraph;
updatedSituationGraph.reasoningState = buildReasoningState(
updatedSituationGraph,
);
deterministicSelection = selectActiveUnknownCandidate(
updatedSituationGraph,
updatedSituationGraph.resolvedNodeIds || [],
);
updatedSituationGraph.activeUnknownNodeId =
deterministicSelection?.status === "selected"
? deterministicSelection.nodeId
: null;
updatedSituationGraph.currentSummary = describeGraph(updatedSituationGraph);
const selectedNode =
deterministicSelection?.status === "selected"
? findNodeById(updatedSituationGraph, deterministicSelection.nodeId)
: null;
const formulatedQuestion = selectedNode
? formulateQuestion({
node: selectedNode,
graph: updatedSituationGraph,
context: { selectionState: deterministicSelection },
})
: null;
return {
success: true,
updatedSituationGraph,
deterministicSelection,
selectedQuestion:
deterministicSelection?.status === "ambiguous"
? {
id: "q_tie_resolution",
...formulateTieResolutionQuestion({ graph: updatedSituationGraph }),
nodeId: null,
tiedCandidateIds: deterministicSelection.tiedCandidateIds,
}
: deterministicSelection?.status === "selected" && formulatedQuestion
? {
nodeId: deterministicSelection.nodeId,
question:
formulatedQuestion.question || deterministicSelection.question,
reason: formulatedQuestion.reason,
strategy: formulatedQuestion.strategy,
investigationStrategy: formulatedQuestion.investigationStrategy,
questionComplexity: formulatedQuestion.questionComplexity,
plainLanguageNormalisations:
formulatedQuestion.plainLanguageNormalisations,
}
: null,
atomicityAssessment: decompositionResult.atomicityAssessment,
decompositionPerformed:
decompositionResult.decompositionAttempted &&
decompositionResult.decompositionAccepted,
decompositionAttempted: decompositionResult.decompositionAttempted,
selectedUnknownBefore: decompositionResult.selectedUnknownBefore,
selectedUnknownAfter: deterministicSelection?.nodeId ?? null,
questionComplexityAssessment:
formulatedQuestion?.questionComplexity ?? null,
};
}
function runDeterministicDecomposition({
graphSnapshot,
proposalSnapshot,
lib/graph/orchestrator.js
+66 -25
View File
@@ -13,10 +13,14 @@ import {
updateCaseRequestSchema,
} from "./schema.js";
import { buildInitialGraph, describeGraph } from "./builder.js";
import { applyValidatedProposal } from "./apply-proposal.js";
import {
applyValidatedProposal,
determineGraphBackedQuestion,
} from "./apply-proposal.js";
import { buildGraphUpdatePrompt } from "./prompt-builder.js";
import {
buildReasoningState,
formulateQuestion,
formulateTieResolutionQuestion,
} from "./question-formulator.js";
import { parseGraphUpdateProposal } from "./update-proposal.js";
@@ -41,6 +45,13 @@ function buildDiagnostics({
graph,
graphReferenceValidation,
unknownSelectionExplanation,
reconstructionQuestion,
reconstructionQuestionAccepted,
reconstructionQuestionRejectionReasons,
finalGraphBackedQuestion,
selectedUnknownNodeId,
decompositionApplied,
questionComplexityAssessment,
}) {
return {
promptVersion: analysis?.promptVersion ?? null,
@@ -54,6 +65,14 @@ function buildDiagnostics({
compatibilityChanges: analysis?.compatibilityChanges ?? [],
compatibilityWarnings: analysis?.compatibilityWarnings ?? [],
unknownSelectionExplanation: unknownSelectionExplanation ?? null,
reconstructionQuestion: reconstructionQuestion ?? null,
reconstructionQuestionAccepted: reconstructionQuestionAccepted ?? null,
reconstructionQuestionRejectionReasons:
reconstructionQuestionRejectionReasons ?? [],
finalGraphBackedQuestion: finalGraphBackedQuestion ?? null,
selectedUnknownNodeId: selectedUnknownNodeId ?? null,
decompositionApplied: decompositionApplied ?? false,
questionComplexityAssessment: questionComplexityAssessment ?? null,
};
}
@@ -263,23 +282,11 @@ export async function startCase(body) {
});
const currentSummary = describeGraph(initialGraph);
const deterministicSelection = selectActiveUnknownCandidate(
{
...initialGraph,
resolvedNodeIds: [],
},
[],
);
const activeUnknownNodeId =
deterministicSelection?.status === "selected"
? deterministicSelection.nodeId
: null;
const situationGraph = makeGraph({
const initialSituationGraph = makeGraph({
centralStatement: scenario,
nodes: initialGraph.nodes,
edges: initialGraph.edges,
activeUnknownNodeId,
activeUnknownNodeId: null,
resolvedNodeIds: [],
currentSummary,
reasoningState: buildReasoningState({
@@ -290,17 +297,20 @@ export async function startCase(body) {
}),
});
situationGraphSchema.parse(situationGraph);
situationGraphSchema.parse(initialSituationGraph);
const graphReferenceValidation = validateGraphReferences(situationGraph);
const selectedQuestion =
deterministicSelection?.status === "ambiguous"
? {
id: "q_tie_resolution",
...formulateTieResolutionQuestion({ graph: situationGraph }),
tiedCandidateIds: deterministicSelection.tiedCandidateIds,
}
: (analysis.nextQuestion ?? null);
const graphReferenceValidation = validateGraphReferences(
initialSituationGraph,
);
const initialQuestionResult = determineGraphBackedQuestion({
situationGraph: initialSituationGraph,
});
const situationGraph = initialQuestionResult.success
? initialQuestionResult.updatedSituationGraph
: initialSituationGraph;
const selectedQuestion = initialQuestionResult.success
? initialQuestionResult.selectedQuestion
: null;
const unknownSelectionExplanation = buildUnknownSelectionDiagnostics(
situationGraph,
[],
@@ -315,6 +325,22 @@ export async function startCase(body) {
graph: situationGraph,
graphReferenceValidation,
unknownSelectionExplanation,
reconstructionQuestion: analysis.nextQuestion?.question ?? null,
reconstructionQuestionAccepted: false,
reconstructionQuestionRejectionReasons:
analysis.nextQuestion?.question != null
? [
"reconstruction_question_not_authoritative",
"graph_backed_pipeline_required",
]
: [],
finalGraphBackedQuestion: selectedQuestion?.question ?? null,
selectedUnknownNodeId:
initialQuestionResult.selectedUnknownAfter ?? null,
decompositionApplied:
initialQuestionResult.decompositionPerformed ?? false,
questionComplexityAssessment:
initialQuestionResult.questionComplexityAssessment ?? null,
}),
validationErrors: graphReferenceValidation.errors,
statusCode: 500,
@@ -330,6 +356,21 @@ export async function startCase(body) {
graph: situationGraph,
graphReferenceValidation,
unknownSelectionExplanation,
reconstructionQuestion: analysis.nextQuestion?.question ?? null,
reconstructionQuestionAccepted: false,
reconstructionQuestionRejectionReasons:
analysis.nextQuestion?.question != null
? [
"reconstruction_question_not_authoritative",
"graph_backed_pipeline_required",
]
: [],
finalGraphBackedQuestion: selectedQuestion?.question ?? null,
selectedUnknownNodeId: initialQuestionResult.selectedUnknownAfter ?? null,
decompositionApplied:
initialQuestionResult.decompositionPerformed ?? false,
questionComplexityAssessment:
initialQuestionResult.questionComplexityAssessment ?? null,
}),
};
}
tests/graph/orchestrator.test.js
+85 -6
View File
@@ -56,6 +56,41 @@ function makeAnalysisResult(overrides = {}) {
};
}
function makeCommercialAnalysisResult(overrides = {}) {
return makeAnalysisResult({
reconstruction: {
summary:
"A new reasoning method may become a commercial product, but problem existence and value remain unresolved.",
actors: [],
systemsOrObjects: [],
expectedStates: [],
observedStates: [],
differences: [],
knownTransitions: [],
unexplainedTransitions: [],
contradictions: [],
importantUnknowns: [
{
id: "unk-commercial",
label:
"Commercial justification for whether continuing development is commercially justified",
description:
"Need to know whether this solves a genuine problem, whether people would value it enough to pay for it, and whether it is commercially justified before continuing development.",
confidence: "high",
},
],
plausibleInterpretations: [],
},
nextQuestion: {
id: "q-commercial",
question:
"What specific validation metrics, pilot feedback, or competitive benchmarking results have you collected to measure whether the method solves a recognized problem and how target users evaluate its practical utility compared to existing tools?",
reason: "Model-proposed broad validation question",
},
...overrides,
});
}
function makeUpdateGraph() {
const unknown = makeNode({
id: "n-unknown",
@@ -401,9 +436,15 @@ describe("lib/graph/orchestrator startCase", () => {
});
});
it("returns null selectedQuestion when analysis has no nextQuestion", async () => {
it("returns null selectedQuestion when neither analysis nor graph path yields a question", async () => {
mockAnalyseScenario.mockResolvedValue(
makeAnalysisResult({ nextQuestion: undefined }),
makeAnalysisResult({
nextQuestion: undefined,
reconstruction: {
...makeAnalysisResult().reconstruction,
importantUnknowns: [],
},
}),
);
const { startCase } = await import("@/lib/graph/orchestrator.js");
@@ -413,6 +454,40 @@ describe("lib/graph/orchestrator startCase", () => {
expect(result.selectedQuestion).toBeNull();
});
it("uses the graph-backed question path instead of the reconstruction nextQuestion on startCase", async () => {
mockAnalyseScenario.mockResolvedValue(makeCommercialAnalysisResult());
const { startCase } = await import("@/lib/graph/orchestrator.js");
const result = await startCase({
scenario:
"I have developed a new reasoning method that aims to help people determine whether they have enough justified confidence to make a decision.",
});
expect(result.success).toBe(true);
expect(result.selectedQuestion?.question).toBe(
"Who experiences this problem?",
);
expect(result.selectedQuestion?.nodeId).toBe(
result.situationGraph.activeUnknownNodeId,
);
expect(result.selectedQuestion?.question).not.toContain(
"validation metrics, pilot feedback, or competitive benchmarking",
);
expect(result.diagnostics.reconstructionQuestion).toContain(
"validation metrics, pilot feedback, or competitive benchmarking",
);
expect(result.diagnostics.reconstructionQuestionAccepted).toBe(false);
expect(result.diagnostics.reconstructionQuestionRejectionReasons).toContain(
"graph_backed_pipeline_required",
);
expect(result.diagnostics.finalGraphBackedQuestion).toBe(
"Who experiences this problem?",
);
expect(result.diagnostics.selectedUnknownNodeId).toBe(
result.situationGraph.activeUnknownNodeId,
);
});
it("includes compatibility diagnostics when provided by analysis", async () => {
mockAnalyseScenario.mockResolvedValue(
makeAnalysisResult({
@@ -1093,16 +1168,20 @@ describe("lib/graph/orchestrator startCase", () => {
);
});
it("startCase behaviour remains unchanged", async () => {
it("startCase no longer copies analysis nextQuestion directly when a graph-backed question exists", async () => {
mockAnalyseScenario.mockResolvedValue(makeAnalysisResult());
const { startCase } = await import("@/lib/graph/orchestrator.js");
const result = await startCase({ scenario: "Scenario text" });
expect(result.success).toBe(true);
expect(result.selectedQuestion).toEqual({
id: "q-1",
question: "What denominator is being used for the complaint rate?",
expect(result.selectedQuestion).toMatchObject({
nodeId: result.situationGraph.activeUnknownNodeId,
question:
"What would clarify need the denominator for complaint rate in this situation?",
});
expect(result.diagnostics.reconstructionQuestion).toBe(
"What denominator is being used for the complaint rate?",
);
});
});