fix: make graph-backed questions authoritative

This commit is contained in:
2026-08-03 09:13:52 +01:00
parent ef04b9e494
commit db994d7764
4 changed files with 285 additions and 31 deletions
+85 -6
View File
@@ -56,6 +56,41 @@ function makeAnalysisResult(overrides = {}) {
};
}
function makeCommercialAnalysisResult(overrides = {}) {
return makeAnalysisResult({
reconstruction: {
summary:
"A new reasoning method may become a commercial product, but problem existence and value remain unresolved.",
actors: [],
systemsOrObjects: [],
expectedStates: [],
observedStates: [],
differences: [],
knownTransitions: [],
unexplainedTransitions: [],
contradictions: [],
importantUnknowns: [
{
id: "unk-commercial",
label:
"Commercial justification for whether continuing development is commercially justified",
description:
"Need to know whether this solves a genuine problem, whether people would value it enough to pay for it, and whether it is commercially justified before continuing development.",
confidence: "high",
},
],
plausibleInterpretations: [],
},
nextQuestion: {
id: "q-commercial",
question:
"What specific validation metrics, pilot feedback, or competitive benchmarking results have you collected to measure whether the method solves a recognized problem and how target users evaluate its practical utility compared to existing tools?",
reason: "Model-proposed broad validation question",
},
...overrides,
});
}
function makeUpdateGraph() {
const unknown = makeNode({
id: "n-unknown",
@@ -401,9 +436,15 @@ describe("lib/graph/orchestrator startCase", () => {
});
});
it("returns null selectedQuestion when analysis has no nextQuestion", async () => {
it("returns null selectedQuestion when neither analysis nor graph path yields a question", async () => {
mockAnalyseScenario.mockResolvedValue(
makeAnalysisResult({ nextQuestion: undefined }),
makeAnalysisResult({
nextQuestion: undefined,
reconstruction: {
...makeAnalysisResult().reconstruction,
importantUnknowns: [],
},
}),
);
const { startCase } = await import("@/lib/graph/orchestrator.js");
@@ -413,6 +454,40 @@ describe("lib/graph/orchestrator startCase", () => {
expect(result.selectedQuestion).toBeNull();
});
it("uses the graph-backed question path instead of the reconstruction nextQuestion on startCase", async () => {
mockAnalyseScenario.mockResolvedValue(makeCommercialAnalysisResult());
const { startCase } = await import("@/lib/graph/orchestrator.js");
const result = await startCase({
scenario:
"I have developed a new reasoning method that aims to help people determine whether they have enough justified confidence to make a decision.",
});
expect(result.success).toBe(true);
expect(result.selectedQuestion?.question).toBe(
"Who experiences this problem?",
);
expect(result.selectedQuestion?.nodeId).toBe(
result.situationGraph.activeUnknownNodeId,
);
expect(result.selectedQuestion?.question).not.toContain(
"validation metrics, pilot feedback, or competitive benchmarking",
);
expect(result.diagnostics.reconstructionQuestion).toContain(
"validation metrics, pilot feedback, or competitive benchmarking",
);
expect(result.diagnostics.reconstructionQuestionAccepted).toBe(false);
expect(result.diagnostics.reconstructionQuestionRejectionReasons).toContain(
"graph_backed_pipeline_required",
);
expect(result.diagnostics.finalGraphBackedQuestion).toBe(
"Who experiences this problem?",
);
expect(result.diagnostics.selectedUnknownNodeId).toBe(
result.situationGraph.activeUnknownNodeId,
);
});
it("includes compatibility diagnostics when provided by analysis", async () => {
mockAnalyseScenario.mockResolvedValue(
makeAnalysisResult({
@@ -1093,16 +1168,20 @@ describe("lib/graph/orchestrator startCase", () => {
);
});
it("startCase behaviour remains unchanged", async () => {
it("startCase no longer copies analysis nextQuestion directly when a graph-backed question exists", async () => {
mockAnalyseScenario.mockResolvedValue(makeAnalysisResult());
const { startCase } = await import("@/lib/graph/orchestrator.js");
const result = await startCase({ scenario: "Scenario text" });
expect(result.success).toBe(true);
expect(result.selectedQuestion).toEqual({
id: "q-1",
question: "What denominator is being used for the complaint rate?",
expect(result.selectedQuestion).toMatchObject({
nodeId: result.situationGraph.activeUnknownNodeId,
question:
"What would clarify need the denominator for complaint rate in this situation?",
});
expect(result.diagnostics.reconstructionQuestion).toBe(
"What denominator is being used for the complaint rate?",
);
});
});