From 6bc8be6edbcce38000dea48408e5b8fc32a8d994 Mon Sep 17 00:00:00 2001 From: robbond Date: Fri, 9 Oct 2026 07:46:19 +0100 Subject: [PATCH] wip(confidence-engine): checkpoint open-question presentation work --- components/reasoning-workspace.jsx | 5 +- lib/analysis.js | 13 +- lib/graph/builder.js | 1 + lib/graph/orchestrator.js | 4 + lib/graph/schema.js | 2 + lib/llm/provider.js | 22 +- lib/presentation/facilitator-view-adapter.js | 14 +- .../open-question-presentation.js | 63 ++++++ lib/reconstruction/schema.js | 30 ++- prompts/reconstruct-v0.2.md | 3 +- prompts/reconstruct-v0.3.md | 3 +- prompts/reconstruct-v0.4.md | 4 +- prompts/reconstruct-v0.5.md | 4 +- tests/facilitator-view-adapter.test.js | 39 +++- .../graph/open-question-presentation.test.js | 204 ++++++++++++++++++ tests/graph/orchestrator.test.js | 30 +++ tests/llm/provider.test.js | 137 +++++++++++- tests/reconstruction/compatibility.test.js | 114 +++++++++- .../open-question-presentation.test.js | 142 ++++++++++++ 19 files changed, 812 insertions(+), 22 deletions(-) create mode 100644 lib/reconstruction/open-question-presentation.js create mode 100644 tests/graph/open-question-presentation.test.js create mode 100644 tests/reconstruction/open-question-presentation.test.js diff --git a/components/reasoning-workspace.jsx b/components/reasoning-workspace.jsx index 647d83a..f168bae 100644 --- a/components/reasoning-workspace.jsx +++ b/components/reasoning-workspace.jsx @@ -1890,6 +1890,7 @@ export default function ReasoningWorkspace({ focusedInvestigations[node.id] && focusedInvestigations[node.id].question ); + const openQuestion = node.openQuestion || node.label; function handleNodeClick(n) { if (hasCompleted && selectedPresentationItemId === n.id) { @@ -1907,8 +1908,8 @@ export default function ReasoningWorkspace({ style={{ cursor: "pointer" }} className="w-full text-left rounded-lg border border-gray-200 bg-white px-5 py-4 transition hover:border-gray-300 hover:bg-gray-50" > - {node.label} - {node.description && node.description !== node.label && ( + {openQuestion} + {node.description && node.description !== openQuestion && (

{node.description}

)} {!isFocused && Unclear} diff --git a/lib/analysis.js b/lib/analysis.js index 5f36965..475f2b9 100644 --- a/lib/analysis.js +++ b/lib/analysis.js @@ -12,9 +12,12 @@ import { } from "../lib/reconstruction/prompt.js"; import { normaliseAnalysisResponse } from "../lib/reconstruction/compatibility.js"; import { + reconstructionGenerationJsonSchema, + reconstructionGenerationSchema, reconstructionV2Schema, reconstructionSchema as reconstructionV1Schema, } from "../lib/reconstruction/schema.js"; +import { validateReconstructionOpenQuestions } from "../lib/reconstruction/open-question-presentation.js"; const MAX_SCENARIO_LENGTH = 10000; @@ -80,6 +83,8 @@ export async function analyseScenario(scenario, opts = {}) { const providerResult = await provider.generateReconstruction( promptObj.prompt, reconstructionModelName, + reconstructionGenerationJsonSchema, + { outputZodSchema: reconstructionGenerationSchema }, ); if ( providerResult && @@ -201,6 +206,9 @@ function buildCompatibilityDiagnostics(compatibility) { } function buildSuccessResultV2(data, model, duration, version, compatibility) { + const presentationValidation = validateReconstructionOpenQuestions( + data.reconstruction, + ); return { success: true, validationStatus: "valid", @@ -209,10 +217,13 @@ function buildSuccessResultV2(data, model, duration, version, compatibility) { rawResponse: JSON.stringify(data).slice(0, 3000), promptVersion: version, inputClassification: data.inputClassification, - reconstruction: data.reconstruction, + reconstruction: presentationValidation.reconstruction, evidence: data.evidence, nextQuestion: data.nextQuestion, errors: undefined, + presentationValidationStatus: + presentationValidation.issues.length > 0 ? "partial" : "valid", + presentationValidationIssues: presentationValidation.issues, ...buildCompatibilityDiagnostics(compatibility), }; } diff --git a/lib/graph/builder.js b/lib/graph/builder.js index 61dd8c7..322a31a 100644 --- a/lib/graph/builder.js +++ b/lib/graph/builder.js @@ -184,6 +184,7 @@ export function buildInitialGraph(analysisData) { null, unk.confidence || "low", ); + if (unk.question) node.openQuestion = unk.question; mapSourceId(unk.id, node); unknownNodes.push(node); } diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index 86cf066..c0b5bbf 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -88,6 +88,10 @@ function buildDiagnostics({ compatibilityApplied: analysis?.compatibilityApplied ?? false, compatibilityChanges: analysis?.compatibilityChanges ?? [], compatibilityWarnings: analysis?.compatibilityWarnings ?? [], + presentationValidationStatus: + analysis?.presentationValidationStatus ?? "not_applicable", + presentationValidationIssues: + analysis?.presentationValidationIssues ?? [], unknownSelectionExplanation: unknownSelectionExplanation ?? null, reconstructionQuestion: reconstructionQuestion ?? null, reconstructionQuestionAccepted: reconstructionQuestionAccepted ?? null, diff --git a/lib/graph/schema.js b/lib/graph/schema.js index 110de8a..0654983 100644 --- a/lib/graph/schema.js +++ b/lib/graph/schema.js @@ -57,6 +57,7 @@ export const situationNodeSchema = z.object({ id: z.string().min(1), label: z.string().min(1), description: z.string().min(1), + openQuestion: z.string().min(1).optional(), kind: z.enum(Object.values(SituationKind)), status: z.enum(Object.values(SituationStatus)), confidence: z.enum(Object.values(ConfidenceLevel)), @@ -259,6 +260,7 @@ export function makeNode(opts) { id, label: opts.label, description: opts.description ?? opts.label, + openQuestion: opts.openQuestion, kind: opts.kind ?? "observation", status: opts.status ?? "unknown", confidence: opts.confidence ?? "medium", diff --git a/lib/llm/provider.js b/lib/llm/provider.js index 5ceaf7e..e3caaec 100644 --- a/lib/llm/provider.js +++ b/lib/llm/provider.js @@ -546,10 +546,15 @@ class OpenAIReconstructionProvider { this.fetchImpl = fetchImpl; } - async generateReconstruction(prompt, modelName = "gpt-5.6-terra", outputSchema, { signal } = {}) { + async generateReconstruction( + prompt, + modelName = "gpt-5.6-terra", + outputSchema, + { signal, outputZodSchema } = {}, + ) { if (!this.apiKey) throw new Error("OPENAI_API_KEY is not set"); const structuredOutputSchema = outputSchema - ? createOpenAIStrictSchema(outputSchema, null) + ? createOpenAIStrictSchema(outputSchema, outputZodSchema ?? null) : openAIReconstructionJsonSchema; const format = { @@ -594,10 +599,17 @@ class OpenAIReconstructionProvider { throw error; } + const recoveredResponse = recoverJson(outputText); + const normalisedResponse = outputSchema && !outputZodSchema + ? recoveredResponse + : normaliseOpenAITransportResponse( + recoveredResponse, + outputZodSchema ?? reconstructionV2Schema, + outputSchema ?? reconstructionJsonSchema, + ); + return { - response: outputSchema - ? recoverJson(outputText) - : normaliseOpenAITransportResponse(recoverJson(outputText)), + response: normalisedResponse, providerApiPath: "/v1/responses", }; } diff --git a/lib/presentation/facilitator-view-adapter.js b/lib/presentation/facilitator-view-adapter.js index 8859788..3d4409a 100644 --- a/lib/presentation/facilitator-view-adapter.js +++ b/lib/presentation/facilitator-view-adapter.js @@ -274,8 +274,14 @@ export function buildFacilitatorViewModel(_ref) { // Scaffold items are entirely suppressed from user-facing sections. if (semanticRole === "scaffolding") continue; + // An unknown's openQuestion is a model-authored presentation projection. + // Its label and description remain the canonical uncertainty representation. + var displayText = + node.kind === "unknown" && !resolved && node.openQuestion + ? node.openQuestion.trim() + : displayResult.displayText; var entry = { - text: displayResult.displayText, + text: displayText, normalised: normaliseText(displayResult.displayText), kind: node.kind, semanticRole: semanticRole, @@ -410,8 +416,10 @@ export function buildFacilitatorViewModel(_ref) { if (!a.isActiveUnknown && b.isActiveUnknown) return 1; // Selected-question target: match by normalised text - if (sqText && a.normalised === sqText && b.normalised !== sqText) return -1; - if (sqText && a.normalised !== sqText && b.normalised === sqText) return 1; + var aMatchesSelectedQuestion = sqText && normaliseText(a.text) === sqText; + var bMatchesSelectedQuestion = sqText && normaliseText(b.text) === sqText; + if (aMatchesSelectedQuestion && !bMatchesSelectedQuestion) return -1; + if (!aMatchesSelectedQuestion && bMatchesSelectedQuestion) return 1; // Explicit priority fields where available in the graph var pa = a.priority != null ? a.priority : null; diff --git a/lib/reconstruction/open-question-presentation.js b/lib/reconstruction/open-question-presentation.js new file mode 100644 index 0000000..561b10c --- /dev/null +++ b/lib/reconstruction/open-question-presentation.js @@ -0,0 +1,63 @@ +const ISSUE_MESSAGES = { + question_missing: "No user-facing Open Question was generated.", + question_empty: "The generated Open Question was empty.", + question_not_interrogative: "The generated Open Question was not a direct interrogative.", + question_indirect_proposition: + "The generated Open Question used an indirect uncertainty proposition.", +}; + +/** + * Narrow presentation validation only. This does not attempt to prove semantic + * equivalence with the canonical uncertainty and never rewrites either value. + */ +export function validateOpenQuestionCandidate(candidate) { + if (candidate === undefined || candidate === null) { + return { valid: false, category: "question_missing" }; + } + + if (typeof candidate !== "string" || candidate.trim().length === 0) { + return { valid: false, category: "question_empty" }; + } + + const question = candidate.trim(); + if (/^whether\b/i.test(question)) { + return { valid: false, category: "question_indirect_proposition" }; + } + + if (!question.endsWith("?")) { + return { valid: false, category: "question_not_interrogative" }; + } + + return { valid: true, question }; +} + +/** + * Remove invalid presentation candidates without changing canonical descriptions, + * IDs, ordering, relationships, or any other reconstruction content. + */ +export function validateReconstructionOpenQuestions(reconstruction) { + const issues = []; + const importantUnknowns = (reconstruction?.importantUnknowns ?? []).map( + (unknown, index) => { + const validation = validateOpenQuestionCandidate(unknown.question); + if (validation.valid) { + return { ...unknown, question: validation.question }; + } + + const { question: _invalidQuestion, ...canonicalUnknown } = unknown; + issues.push({ + category: validation.category, + path: ["reconstruction", "importantUnknowns", index, "question"], + unknownId: unknown.id, + message: ISSUE_MESSAGES[validation.category], + severity: "nonfatal", + }); + return canonicalUnknown; + }, + ); + + return { + reconstruction: { ...reconstruction, importantUnknowns }, + issues, + }; +} \ No newline at end of file diff --git a/lib/reconstruction/schema.js b/lib/reconstruction/schema.js index ce13010..96cf9b8 100644 --- a/lib/reconstruction/schema.js +++ b/lib/reconstruction/schema.js @@ -144,6 +144,21 @@ const reconstructionRelationshipSchema = z.object({ confidence: confidenceEnum, }); +const importantUnknownSchema = itemSchemaV1.extend({ + // Canonical uncertainty meaning remains in description. question is an + // optional, model-authored user-facing interrogative for the Open Questions UI. + // Keep this tolerant for historical data and nonfatal presentation validation. + question: z.string().optional(), +}); + +const generatedImportantUnknownSchema = itemSchemaV1.extend({ + // New generation requests require a distinct presentation candidate. More + // nuanced interrogative validation is intentionally performed locally after + // canonical reconstruction validation rather than encoded as provider-specific + // JSON Schema grammar. + question: z.string().trim().min(1), +}); + const reconstructionSchemaV2 = z.object({ summary: z.string().min(1), actors: z.array(itemSchemaV1), @@ -167,7 +182,7 @@ const reconstructionSchemaV2 = z.object({ }), ), contradictions: z.array(itemSchemaV1), - importantUnknowns: z.array(itemSchemaV1), + importantUnknowns: z.array(importantUnknownSchema), plausibleInterpretations: z.array( z.object({ id: z.string().min(1), @@ -205,6 +220,19 @@ export const reconstructionV2Schema = z.object({ nextQuestion: nextQuestionSchema, }); +// Generation-only contract. Durable reconstruction parsing remains tolerant so +// historical snapshots and otherwise valid canonical reconstructions do not fail +// solely because presentation wording is absent or malformed. +export const reconstructionGenerationSchema = reconstructionV2Schema.extend({ + reconstruction: reconstructionSchemaV2.extend({ + importantUnknowns: z.array(generatedImportantUnknownSchema), + }), +}); + +export const reconstructionGenerationJsonSchema = z.toJSONSchema( + reconstructionGenerationSchema, +); + // Outer wrapper for API return (includes diagnostics + v0.2 data) export const analyseResponseV2Schema = z.object({ inputClassification: inputClassificationSchema.optional(), diff --git a/prompts/reconstruct-v0.2.md b/prompts/reconstruct-v0.2.md index 6d0f7dd..413901b 100644 --- a/prompts/reconstruct-v0.2.md +++ b/prompts/reconstruct-v0.2.md @@ -19,6 +19,7 @@ You are a neutral analyst performing evidence-based situation reconstruction. 7. Distinguish: what was said / what it may mean / why it may have been said. 8. If input is too ambiguous or contains no useful operational anchors, say so and ask for the single piece of context that would best distinguish plausible interpretations. +9. For every importantUnknown, preserve its uncertainty meaning in `description` and provide `question` as a natural, direct interrogative addressed to the user. Retain all scope, conditions, timeframes, and qualifications without adding assumptions or an implied conclusion. Do not use fragments or indirect proposition forms such as "Whether …". ## Confidence scale @@ -83,7 +84,7 @@ Return a JSON object with exactly these four top-level keys (use **camelCase**): "knownTransitions": [{"id": "...", "description": "...", "confidence": "", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}], "unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "", "entity": "...", "previousState": "...", "currentState": "..."}], "contradictions": [{"id": "...", "description": "...", "confidence": ""}], - "importantUnknowns": [{"id": "...", "description": "...", "confidence": ""}], + "importantUnknowns": [{"id": "...", "description": "", "question": "", "confidence": ""}], "plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": [""], "assumptionsRequired": [], "confidence": ""}] }, "evidence": [ diff --git a/prompts/reconstruct-v0.3.md b/prompts/reconstruct-v0.3.md index 884515a..daddc97 100644 --- a/prompts/reconstruct-v0.3.md +++ b/prompts/reconstruct-v0.3.md @@ -20,6 +20,7 @@ You are a neutral analyst performing evidence-based situation reconstruction. 8. Distinguish: what was said / what it may mean / why it may have been said. 9. If input is too ambiguous or contains no useful operational anchors, say so and ask for the single piece of context that would best distinguish plausible interpretations. +9a. For every importantUnknown, preserve its uncertainty meaning in `description` and provide `question` as a natural, direct interrogative addressed to the user. Retain all scope, conditions, timeframes, and qualifications without adding assumptions or an implied conclusion. Do not use fragments or indirect proposition forms such as "Whether …". 10. Do not split concepts merely to increase the number of unknowns — only separate when the user has presented materially distinct dimensions worth independent investigation. @@ -123,7 +124,7 @@ Return a JSON object with exactly these four top-level keys (use **camelCase**): "knownTransitions": [{"id": "...", "description": "...", "confidence": "", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}], "unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "", "entity": "...", "previousState": "...", "currentState": "..."}], "contradictions": [{"id": "...", "description": "...", "confidence": ""}], - "importantUnknowns": [{"id": "...", "description": "...", "confidence": ""}], + "importantUnknowns": [{"id": "...", "description": "", "question": "", "confidence": ""}], "plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": [""], "assumptionsRequired": [], "confidence": ""}] }, "evidence": [ diff --git a/prompts/reconstruct-v0.4.md b/prompts/reconstruct-v0.4.md index ff8de7f..461435f 100644 --- a/prompts/reconstruct-v0.4.md +++ b/prompts/reconstruct-v0.4.md @@ -29,6 +29,8 @@ You are a neutral analyst performing evidence-based situation reconstruction. 9. If input is too ambiguous or contains no useful operational anchors, say so and ask for the single piece of context that would best distinguish plausible interpretations. +9a. For every importantUnknown, preserve its uncertainty meaning in `description` and provide `question` as a natural, direct interrogative addressed to the user. Retain all scope, conditions, timeframes, and qualifications without adding assumptions or an implied conclusion. Do not use fragments or indirect proposition forms such as "Whether …". + 10. Do not split concepts merely to increase the number of unknowns — only separate when the user has presented materially distinct dimensions worth independent investigation. ## Preserve supplied relationships @@ -139,7 +141,7 @@ Return a JSON object with exactly these four top-level keys (use **camelCase**): "knownTransitions": [{"id": "...", "description": "...", "confidence": "", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}], "unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "", "entity": "...", "previousState": "...", "currentState": "..."}], "contradictions": [{"id": "...", "description": "...", "confidence": ""}], - "importantUnknowns": [{"id": "...", "description": "...", "confidence": ""}], + "importantUnknowns": [{"id": "...", "description": "", "question": "", "confidence": ""}], "plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": [""], "assumptionsRequired": [], "confidence": ""}] }, "evidence": [ diff --git a/prompts/reconstruct-v0.5.md b/prompts/reconstruct-v0.5.md index 3de9e53..b83522a 100644 --- a/prompts/reconstruct-v0.5.md +++ b/prompts/reconstruct-v0.5.md @@ -26,6 +26,8 @@ You are a neutral analyst performing evidence-based situation reconstruction. 7. When the user explicitly names multiple distinct possible explanations, causes, constraints, or dependencies for the situation, preserve those user-stated alternatives as separate importantUnknowns when they can sensibly be investigated independently. Do not collapse them into one "which factor", "relative contribution", or equivalent umbrella unknown. Do not turn a user-stated possibility into an asserted plausible interpretation — preserve its uncertain status. Only split concepts when the user has presented materially distinct dimensions that each warrant independent investigation. +7a. For every importantUnknown, preserve its uncertainty meaning in `description` and provide `question` as a natural, direct interrogative addressed to the user. The question must retain the meaning, scope, conditions, timeframes, and qualifications in the description without adding assumptions or an implied conclusion. Do not use fragments or indirect proposition forms such as "Whether …". This applies across domains. + 8. Distinguish: what was said / what it may mean / why it may have been said. 9. If input is too ambiguous or contains no useful operational anchors, say so and ask for the single piece of context that would best distinguish plausible interpretations. @@ -152,7 +154,7 @@ Return a JSON object with exactly these four top-level keys (use **camelCase**): "knownTransitions": [{"id": "...", "description": "...", "confidence": "", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}], "unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "", "entity": "...", "previousState": "...", "currentState": "..."}], "contradictions": [{"id": "...", "description": "...", "confidence": ""}], - "importantUnknowns": [{"id": "...", "description": "...", "confidence": ""}], + "importantUnknowns": [{"id": "...", "description": "", "question": "", "confidence": ""}], "plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": [""], "assumptionsRequired": [], "confidence": ""}], "relationships": [{"id": "...", "fromId": "", "toId": "", "relationship": "", "description": "...", "confidence": ""}] }, diff --git a/tests/facilitator-view-adapter.test.js b/tests/facilitator-view-adapter.test.js index 132f3ed..ecd95b7 100644 --- a/tests/facilitator-view-adapter.test.js +++ b/tests/facilitator-view-adapter.test.js @@ -7,11 +7,13 @@ function mkN(id, label, opts) { var k = (opts && opts.kind) || "unknown"; var s = (opts && opts.status) || (k === "unknown" ? "unknown" : "known"); var c = (opts && opts.confidence) || "low"; - return { + var node = { id, label, description: (opts && opts.description) || label, kind: k, status: s, confidence: c, confidenceAssessment: { evidenceConfidence: c, completenessStatus: "partial", conclusionConfidence: c }, value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], childIds: [] }; + if (opts && opts.openQuestion) node.openQuestion = opts.openQuestion; + return node; } /* ── Filtering tests ─────────────────────────────────────────────── */ @@ -147,6 +149,41 @@ describe("Deduplication", () => { }); // The unresolved one (if any) or the cross-dedup should prevent duplication }); + + it("keeps canonically distinct uncertainties with identical presentation questions", () => { + const sharedQuestion = "What evidence would resolve this uncertainty?"; + const vm = buildFacilitatorViewModel({ + nodes: [ + mkN("u1", "Whether supplier capacity is sufficient for launch", { + openQuestion: sharedQuestion, + }), + mkN("u2", "Whether customer demand is sufficient for launch", { + openQuestion: sharedQuestion, + }), + ], + resolvedIds: new Set(), + }); + + expect(vm.investigating.items.filter((item) => item === sharedQuestion)).toHaveLength(2); + }); + + it("continues to merge genuine canonical duplicates despite different presentation questions", () => { + const vm = buildFacilitatorViewModel({ + nodes: [ + mkN("u1", "Whether supplier capacity is sufficient for launch", { + openQuestion: "Can the supplier support the launch volume?", + }), + mkN("u2", "whether supplier capacity is sufficient for launch!", { + openQuestion: "Is enough supplier capacity available?", + }), + ], + resolvedIds: new Set(), + }); + + expect(vm.investigating.items).toEqual([ + "Can the supplier support the launch volume?", + ]); + }); }); /* ── Ranking tests ───────────────────────────────────────────────── */ diff --git a/tests/graph/open-question-presentation.test.js b/tests/graph/open-question-presentation.test.js new file mode 100644 index 0000000..e492738 --- /dev/null +++ b/tests/graph/open-question-presentation.test.js @@ -0,0 +1,204 @@ +import { describe, expect, it } from "vitest"; +import { buildInitialGraph } from "@/lib/graph/builder.js"; +import { formulateQuestionForTarget } from "@/lib/graph/focused-investigation.js"; +import { makeGraph, makeNodeId } from "@/lib/graph/schema.js"; +import { buildFacilitatorViewModel } from "@/lib/presentation/facilitator-view-adapter.js"; +import { buildPrompt } from "@/lib/reconstruction/prompt.js"; +import { validateReconstructionOpenQuestions } from "@/lib/reconstruction/open-question-presentation.js"; +import { reconstructionV2Schema } from "@/lib/reconstruction/schema.js"; + +function reconstructionWithUnknowns(importantUnknowns) { + return { + inputClassification: { + primaryType: "other", + secondaryTypes: [], + reasoningModes: ["identify_missing_information"], + classificationReason: "The scenario contains unresolved factors.", + confidence: "medium", + }, + reconstruction: { + summary: "A situation with unresolved factors.", + actors: [], + systemsOrObjects: [], + expectedStates: [], + observedStates: [], + differences: [], + knownTransitions: [], + unexplainedTransitions: [], + contradictions: [], + importantUnknowns, + plausibleInterpretations: [], + relationships: [], + }, + evidence: [], + nextQuestion: { + id: "next-question", + question: "What information would clarify the situation?", + targets: [], + reason: "The situation has unresolved factors.", + expectedInformationValue: "medium", + reasoningMode: "identify_missing_information", + }, + }; +} + +const CASES = [ + { + name: "recruitment uncertainty", + description: "Whether skilled production staff can be recruited within the required timeframe", + question: "Can skilled production staff be recruited within the required timeframe?", + terms: ["skilled production staff", "required timeframe"], + }, + { + name: "demand uncertainty", + description: "The level of demand in the North West market needed to justify the expansion cost", + question: "What level of demand in the North West market is needed to justify the expansion cost?", + terms: ["North West market", "justify the expansion cost"], + }, + { + name: "historical uncertainty", + description: "Whether the prior staffing level was sufficient before the expansion", + question: "Was the prior staffing level sufficient before the expansion?", + terms: ["prior staffing level", "before the expansion"], + }, + { + name: "non-business uncertainty", + description: "The time at which the missing hiker last used the eastern trail", + question: "When did the missing hiker last use the eastern trail?", + terms: ["missing hiker", "eastern trail"], + }, +]; + +describe("Open Question presentation", () => { + it.each(["v0.2", "v0.3", "v0.4", "v0.5"])("instructs reconstruction prompt %s to produce a direct Open Question", async (version) => { + const { prompt } = await buildPrompt("A scenario with an unresolved condition.", version); + + expect(prompt).toContain("importantUnknown"); + expect(prompt).toContain("natural, direct interrogative"); + expect(prompt).toContain('"question"'); + expect(prompt).toContain('"Whether …"'); + }); + + it.each(CASES)("preserves $name meaning while displaying a direct interrogative", ({ description, question, terms }) => { + const parsed = reconstructionV2Schema.parse( + reconstructionWithUnknowns([{ id: "unknown", description, question, confidence: "medium" }]), + ); + const graph = buildInitialGraph({ reconstruction: parsed.reconstruction, evidence: [] }); + const unknown = graph.nodes.find((node) => node.kind === "unknown"); + const viewModel = buildFacilitatorViewModel({ nodes: graph.nodes, resolvedIds: new Set() }); + + expect(unknown.label).toBe(description); + expect(unknown.description).toBe(description); + expect(unknown.openQuestion).toBe(question); + expect(viewModel.investigating.items).toContain(question); + expect(question).toMatch(/\?$/); + expect(question).not.toMatch(/^whether\b/i); + expect(question).not.toMatch(/^before whether\b/i); + for (const term of terms) expect(question.toLowerCase()).toContain(term.toLowerCase()); + }); + + it("leaves canonical identity, explicit target selection, and focused formulation unchanged", () => { + const description = CASES[0].description; + const question = CASES[0].question; + const graphData = buildInitialGraph({ + reconstruction: reconstructionWithUnknowns([ + { id: "recruitment", description, question, confidence: "medium" }, + ]).reconstruction, + evidence: [], + }); + const target = graphData.nodes.find((node) => node.kind === "unknown"); + const graph = makeGraph({ + centralStatement: "A manufacturer is considering expansion.", + nodes: graphData.nodes, + edges: graphData.edges, + activeUnknownNodeId: null, + resolvedNodeIds: [], + currentSummary: "Expansion depends on recruitment capacity.", + }); + const before = JSON.stringify(graph); + const result = formulateQuestionForTarget({ situationGraph: graph, targetNodeId: target.id }); + + expect(target.id).toBe(makeNodeId(description)); + expect(target.id).not.toBe(makeNodeId(question)); + expect(result.success).toBe(true); + expect(result.targetNodeId).toBe(target.id); + expect(result.question).toMatch(/\?$/); + expect(result.question.toLowerCase()).toContain("staff"); + expect(JSON.stringify(graph)).toBe(before); + }); + + it("keeps persisted legacy unknowns valid and falls back to canonical text when no display question exists", () => { + const legacyDescription = "Whether the figures are comparable"; + const legacyNode = makeGraph({ + centralStatement: "Compare two figures.", + nodes: [ + { + id: makeNodeId(legacyDescription), + label: legacyDescription, + description: legacyDescription, + kind: "unknown", + status: "unknown", + confidence: "medium", + evidenceIds: [], + dependsOn: [], + affects: [], + parentId: null, + childIds: [], + }, + ], + edges: [], + activeUnknownNodeId: null, + resolvedNodeIds: [], + currentSummary: "The comparison remains unresolved.", + }); + const viewModel = buildFacilitatorViewModel({ nodes: legacyNode.nodes, resolvedIds: new Set() }); + + expect(viewModel.investigating.items).toContain(legacyDescription); + expect(legacyNode.nodes[0]).not.toHaveProperty("openQuestion"); + }); + + it.each([ + ["missing", undefined, false], + ["empty", " ", true], + ["whether fragment", "Whether the facility can recruit workers in time", true], + ["whether fragment with punctuation", "Whether the facility can recruit workers in time?", true], + ["statement", "The facility can recruit workers in time.", true], + ])( + "keeps an invalid %s presentation canonically selectable and investigable", + (_name, question, includeQuestion) => { + const description = + "Whether the facility can recruit skilled workers within the launch timeframe."; + const unknown = { + id: "unknown", + description, + confidence: "medium", + }; + if (includeQuestion) unknown.question = question; + const reconstruction = reconstructionWithUnknowns([unknown]).reconstruction; + const validated = validateReconstructionOpenQuestions(reconstruction); + const graphParts = buildInitialGraph({ + reconstruction: validated.reconstruction, + evidence: [], + }); + const graph = makeGraph({ + centralStatement: "The facility launch depends on workforce availability.", + nodes: graphParts.nodes, + edges: graphParts.edges, + currentSummary: "Workforce availability remains unresolved.", + }); + const graphUnknown = graph.nodes.find((node) => node.kind === "unknown"); + const focused = formulateQuestionForTarget({ + situationGraph: graph, + targetNodeId: graphUnknown.id, + }); + + expect(validated.issues).toHaveLength(1); + expect(graphUnknown.label).toBe(description); + expect(graphUnknown.description).toBe(description); + expect(graphUnknown).not.toHaveProperty("openQuestion"); + expect(graphUnknown.id).toBe(makeNodeId(description)); + expect(focused.success).toBe(true); + expect(focused.targetNodeId).toBe(graphUnknown.id); + }, + ); +}); \ No newline at end of file diff --git a/tests/graph/orchestrator.test.js b/tests/graph/orchestrator.test.js index a4728c3..05f0df8 100644 --- a/tests/graph/orchestrator.test.js +++ b/tests/graph/orchestrator.test.js @@ -55,6 +55,8 @@ function makeAnalysisResult(overrides = {}) { compatibilityApplied: false, compatibilityChanges: [], compatibilityWarnings: [], + presentationValidationStatus: "valid", + presentationValidationIssues: [], ...overrides, }; } @@ -399,9 +401,37 @@ describe("lib/graph/orchestrator startCase", () => { valid: true, errors: [], }); + expect(result.diagnostics.presentationValidationStatus).toBe("valid"); + expect(result.diagnostics.presentationValidationIssues).toEqual([]); expect(downstreamProvider.generateReconstruction).not.toHaveBeenCalled(); }); + it("propagates nonfatal Open Question presentation issues through diagnostics", async () => { + const issue = { + category: "question_indirect_proposition", + path: ["reconstruction", "importantUnknowns", 0, "question"], + unknownId: "unk-1", + message: "The generated Open Question used an indirect uncertainty proposition.", + severity: "nonfatal", + }; + mockAnalyseScenario.mockResolvedValue( + makeAnalysisResult({ + presentationValidationStatus: "partial", + presentationValidationIssues: [issue], + }), + ); + const { startCase } = await import("@/lib/graph/orchestrator.js"); + + const result = await startCase( + { scenario: "Scenario text" }, + { reconstructionOnly: true }, + ); + + expect(result.success).toBe(true); + expect(result.diagnostics.presentationValidationStatus).toBe("partial"); + expect(result.diagnostics.presentationValidationIssues).toEqual([issue]); + }); + it("rejects invalid request input without throwing", async () => { const { startCase } = await import("@/lib/graph/orchestrator.js"); diff --git a/tests/llm/provider.test.js b/tests/llm/provider.test.js index 3b08311..d5d598b 100644 --- a/tests/llm/provider.test.js +++ b/tests/llm/provider.test.js @@ -8,7 +8,11 @@ import { normaliseOpenAITransportResponse, reconstructionJsonSchema, } from "@/lib/llm/provider.js"; -import { reconstructionV2Schema } from "@/lib/reconstruction/schema.js"; +import { + reconstructionGenerationJsonSchema, + reconstructionGenerationSchema, + reconstructionV2Schema, +} from "@/lib/reconstruction/schema.js"; import { focusedDeconstructJsonSchema } from "@/lib/graph/focused-investigation.js"; import { z } from "zod"; @@ -299,6 +303,137 @@ describe("OpenAI reconstruction provider experiment seam", () => { expect(acceptsNull(transition.properties.id)).toBe(false); }); + it("preserves optional transport nullability for the generation-specific schema", async () => { + const fetchImpl = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ output_text: "{}" }), + }); + const provider = createOpenAIReconstructionProvider({ + apiKey: "test-key", + fetchImpl, + }); + + await provider.generateReconstruction( + "prompt", + "test-model", + reconstructionGenerationJsonSchema, + { outputZodSchema: reconstructionGenerationSchema }, + ); + + const request = JSON.parse(fetchImpl.mock.calls[0][1].body); + const reconstruction = request.text.format.schema.properties.reconstruction; + const unknown = reconstruction.properties.importantUnknowns.items; + const unexplained = reconstruction.properties.unexplainedTransitions.items; + + expect(unknown.required).toContain("question"); + expect(unknown.properties.question.minLength).toBe(1); + expect(unexplained.required).toContain("entity"); + expect(unexplained.properties.entity.anyOf).toContainEqual({ type: "null" }); + }); + + it("normalises generation-schema null placeholders before canonical validation", async () => { + const transportResponse = { + inputClassification: { + primaryType: "unexplained_change", + secondaryTypes: null, + reasoningModes: null, + classificationReason: "A reported change requires reconstruction.", + confidence: "high", + }, + reconstruction: { + summary: "A change occurred and its cause remains uncertain.", + actors: [], + systemsOrObjects: [], + expectedStates: [], + observedStates: [], + differences: [], + knownTransitions: [], + unexplainedTransitions: [ + { + id: "transition-1", + description: "The service level declined after the release.", + confidence: "medium", + entity: null, + previousState: null, + currentState: null, + }, + ], + contradictions: [], + importantUnknowns: [ + { + id: "unknown-1", + description: "Whether the release caused the service-level decline", + question: "Did the release cause the service-level decline?", + confidence: "medium", + }, + ], + plausibleInterpretations: [ + { + id: "interpretation-1", + description: "The release may have reduced service capacity.", + supportingEvidenceIds: ["evidence-1"], + assumptionsRequired: null, + confidence: "low", + }, + ], + relationships: null, + }, + evidence: [ + { + id: "evidence-1", + description: "The decline was reported after the release.", + evidenceType: "reported_statement", + source: null, + attribution: null, + confidence: "medium", + importance: "important", + }, + ], + nextQuestion: { + id: "next-1", + question: "What changed in service capacity after the release?", + targets: ["unknown-1"], + reason: "The cause of the decline remains unresolved.", + expectedInformationValue: "high", + reasoningMode: null, + }, + }; + const fetchImpl = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ output_text: JSON.stringify(transportResponse) }), + }); + const provider = createOpenAIReconstructionProvider({ + apiKey: "test-key", + fetchImpl, + }); + + const result = await provider.generateReconstruction( + "prompt", + "test-model", + reconstructionGenerationJsonSchema, + { outputZodSchema: reconstructionGenerationSchema }, + ); + const request = JSON.parse(fetchImpl.mock.calls[0][1].body); + const unexplainedSchema = + request.text.format.schema.properties.reconstruction.properties + .unexplainedTransitions.items; + + expect(unexplainedSchema.required).toContain("entity"); + expect(unexplainedSchema.properties.entity.anyOf).toContainEqual({ type: "null" }); + expect(result.response.inputClassification).not.toHaveProperty("secondaryTypes"); + expect(result.response.inputClassification).not.toHaveProperty("reasoningModes"); + expect(result.response.reconstruction.unexplainedTransitions[0]).not.toHaveProperty("entity"); + expect(result.response.reconstruction.unexplainedTransitions[0]).not.toHaveProperty("previousState"); + expect(result.response.reconstruction.unexplainedTransitions[0]).not.toHaveProperty("currentState"); + expect(result.response.reconstruction.plausibleInterpretations[0]).not.toHaveProperty("assumptionsRequired"); + expect(result.response.reconstruction).not.toHaveProperty("relationships"); + expect(result.response.evidence[0]).not.toHaveProperty("source"); + expect(result.response.evidence[0].attribution).toBeNull(); + expect(result.response.nextQuestion).not.toHaveProperty("reasoningMode"); + expect(reconstructionGenerationSchema.safeParse(result.response).success).toBe(true); + expect(reconstructionV2Schema.safeParse(result.response).success).toBe(true); + }); + it("removes only optional transport null placeholders", () => { const schema = z.object({ optionalText: z.string().optional(), diff --git a/tests/reconstruction/compatibility.test.js b/tests/reconstruction/compatibility.test.js index eb3354a..6d8dc6b 100644 --- a/tests/reconstruction/compatibility.test.js +++ b/tests/reconstruction/compatibility.test.js @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { normaliseAnalysisResponse } from "@/lib/reconstruction/compatibility.js"; +import { reconstructionGenerationJsonSchema } from "@/lib/reconstruction/schema.js"; const mockGenerateReconstruction = vi.fn(); @@ -17,6 +18,7 @@ vi.mock("@/lib/llm/provider.js", () => ({ getProvider: () => ({ generateReconstruction: (...args) => mockGenerateReconstruction(...args), }), + getProviderModelName: () => "test-model", })); vi.mock("@/lib/reconstruction/prompt.js", () => ({ @@ -134,13 +136,117 @@ describe("analyseScenario compatibility", () => { }); expect(result.success).toBe(true); - expect(injectedProvider.generateReconstruction).toHaveBeenCalledWith( - "prompt", - "experiment-model", - ); + expect(injectedProvider.generateReconstruction).toHaveBeenCalledTimes(1); + const [prompt, modelName, outputSchema, options] = + injectedProvider.generateReconstruction.mock.calls[0]; + expect(prompt).toBe("prompt"); + expect(modelName).toBe("experiment-model"); + expect(outputSchema).toEqual(reconstructionGenerationJsonSchema); + expect(options.outputZodSchema.safeParse).toBeTypeOf("function"); + expect( + options.outputZodSchema.safeParse({ + inputClassification: { + primaryType: "other", + secondaryTypes: [], + reasoningModes: [], + classificationReason: "reason", + confidence: "medium", + }, + reconstruction: { + summary: "summary", + actors: [], + systemsOrObjects: [], + expectedStates: [], + observedStates: [], + differences: [], + knownTransitions: [], + unexplainedTransitions: [], + contradictions: [], + importantUnknowns: [], + plausibleInterpretations: [], + relationships: [], + }, + evidence: [], + nextQuestion: { + id: "q1", + question: "What next?", + targets: [], + reason: "reason", + expectedInformationValue: "low", + reasoningMode: "other", + }, + }).success, + ).toBe(true); expect(mockGenerateReconstruction).not.toHaveBeenCalled(); }); + it("preserves canonical reconstruction and reports invalid presentation nonfatally", async () => { + const response = { + inputClassification: { + primaryType: "other", + secondaryTypes: [], + reasoningModes: [], + classificationReason: "reason", + confidence: "medium", + }, + reconstruction: { + summary: "summary", + actors: [], + systemsOrObjects: [], + expectedStates: [], + observedStates: [], + differences: [], + knownTransitions: [], + unexplainedTransitions: [], + contradictions: [], + importantUnknowns: [ + { + id: "unknown-1", + description: "Whether workers can be recruited in time.", + question: "Whether workers can be recruited in time?", + confidence: "high", + }, + ], + plausibleInterpretations: [], + relationships: [], + }, + evidence: [], + nextQuestion: { + id: "q1", + question: "What recruitment evidence exists?", + targets: ["unknown-1"], + reason: "reason", + expectedInformationValue: "high", + reasoningMode: "other", + }, + }; + const injectedProvider = { + generateReconstruction: vi.fn().mockResolvedValue(response), + }; + const { analyseScenario } = await import("@/lib/analysis.js"); + + const result = await analyseScenario("Scenario text", { + reconstructionProvider: injectedProvider, + reconstructionModelName: "experiment-model", + }); + + expect(result.success).toBe(true); + expect(result.validationStatus).toBe("valid"); + expect(result.presentationValidationStatus).toBe("partial"); + expect(result.reconstruction.importantUnknowns[0]).toEqual({ + id: "unknown-1", + description: "Whether workers can be recruited in time.", + confidence: "high", + }); + expect(result.presentationValidationIssues).toEqual([ + expect.objectContaining({ + category: "question_indirect_proposition", + unknownId: "unknown-1", + severity: "nonfatal", + }), + ]); + }); + it("preserves an attempted provider API path on provider failure", async () => { const providerError = new Error("Provider failed"); providerError.providerApiPath = "/api/chat"; diff --git a/tests/reconstruction/open-question-presentation.test.js b/tests/reconstruction/open-question-presentation.test.js new file mode 100644 index 0000000..70330d6 --- /dev/null +++ b/tests/reconstruction/open-question-presentation.test.js @@ -0,0 +1,142 @@ +import { describe, expect, it } from "vitest"; +import { promises as fs } from "node:fs"; +import { join } from "node:path"; +import { + reconstructionGenerationJsonSchema, + reconstructionGenerationSchema, + reconstructionV2Schema, +} from "@/lib/reconstruction/schema.js"; +import { + validateOpenQuestionCandidate, + validateReconstructionOpenQuestions, +} from "@/lib/reconstruction/open-question-presentation.js"; + +function makeAnalysisResponse(questionValue, includeQuestion = true) { + const unknown = { + id: "unknown-workforce", + description: + "Whether necessary skilled production workers can be recruited within the required project timeframe.", + confidence: "high", + }; + if (includeQuestion) unknown.question = questionValue; + + return { + inputClassification: { + primaryType: "other", + secondaryTypes: [], + reasoningModes: [], + classificationReason: "A delivery uncertainty is central to the situation.", + confidence: "high", + }, + reconstruction: { + summary: "Delivery depends on recruiting skilled production workers.", + actors: [], + systemsOrObjects: [], + expectedStates: [], + observedStates: [], + differences: [], + knownTransitions: [], + unexplainedTransitions: [], + contradictions: [], + importantUnknowns: [unknown], + plausibleInterpretations: [], + relationships: [], + }, + evidence: [], + nextQuestion: { + id: "next-1", + question: "What recruitment evidence is available?", + targets: ["unknown-workforce"], + reason: "Recruitment feasibility is unresolved.", + expectedInformationValue: "high", + reasoningMode: "other", + }, + }; +} + +describe("new reconstruction generation contract", () => { + it("requires a non-empty question for every generated important unknown", () => { + expect( + reconstructionGenerationSchema.safeParse( + makeAnalysisResponse(undefined, false), + ).success, + ).toBe(false); + expect( + reconstructionGenerationSchema.safeParse(makeAnalysisResponse(" ")).success, + ).toBe(false); + expect( + reconstructionGenerationSchema.safeParse( + makeAnalysisResponse( + "Can the necessary skilled production workers be recruited within the required project timeframe?", + ), + ).success, + ).toBe(true); + + const unknownSchema = + reconstructionGenerationJsonSchema.properties.reconstruction.properties + .importantUnknowns.items; + expect(unknownSchema.required).toContain("description"); + expect(unknownSchema.required).toContain("question"); + expect(unknownSchema.properties.question.minLength).toBe(1); + }); + + it("keeps the durable compatibility schema tolerant of a missing question", () => { + expect( + reconstructionV2Schema.safeParse(makeAnalysisResponse(undefined, false)).success, + ).toBe(true); + }); +}); + +describe("Open Question presentation validation", () => { + it("accepts a trimmed direct interrogative", () => { + expect( + validateOpenQuestionCandidate( + " Can the necessary workers be recruited in time? ", + ), + ).toEqual({ + valid: true, + question: "Can the necessary workers be recruited in time?", + }); + }); + + it("uses canonical UI fallback without a technical explanation", async () => { + const source = await fs.readFile( + join(process.cwd(), "components/reasoning-workspace.jsx"), + "utf8", + ); + + expect(source).toContain("const openQuestion = node.openQuestion || node.label;"); + expect(source).not.toContain( + "Canonical uncertainty — direct question wording unavailable.", + ); + expect(source).toContain("startFocused(n.id);"); + }); + + it.each([ + ["question_missing", undefined, false], + ["question_empty", " ", true], + ["question_indirect_proposition", "Whether workers can be recruited in time", true], + ["question_indirect_proposition", "Whether workers can be recruited in time?", true], + ["question_not_interrogative", "Workers can be recruited in time.", true], + ])("classifies %s nonfatally", (category, candidate, includeQuestion) => { + const input = makeAnalysisResponse(candidate, includeQuestion).reconstruction; + const canonicalBefore = input.importantUnknowns[0].description; + const idBefore = input.importantUnknowns[0].id; + + const result = validateReconstructionOpenQuestions(input); + + expect(result.reconstruction.importantUnknowns[0]).not.toHaveProperty("question"); + expect(result.reconstruction.importantUnknowns[0].description).toBe( + canonicalBefore, + ); + expect(result.reconstruction.importantUnknowns[0].id).toBe(idBefore); + expect(result.issues).toEqual([ + expect.objectContaining({ + category, + unknownId: idBefore, + severity: "nonfatal", + path: ["reconstruction", "importantUnknowns", 0, "question"], + }), + ]); + }); +}); \ No newline at end of file