wip(confidence-engine): checkpoint open-question presentation work
This commit is contained in:
@@ -1890,6 +1890,7 @@ export default function ReasoningWorkspace({
|
|||||||
focusedInvestigations[node.id] &&
|
focusedInvestigations[node.id] &&
|
||||||
focusedInvestigations[node.id].question
|
focusedInvestigations[node.id].question
|
||||||
);
|
);
|
||||||
|
const openQuestion = node.openQuestion || node.label;
|
||||||
|
|
||||||
function handleNodeClick(n) {
|
function handleNodeClick(n) {
|
||||||
if (hasCompleted && selectedPresentationItemId === n.id) {
|
if (hasCompleted && selectedPresentationItemId === n.id) {
|
||||||
@@ -1907,8 +1908,8 @@ export default function ReasoningWorkspace({
|
|||||||
style={{ cursor: "pointer" }}
|
style={{ cursor: "pointer" }}
|
||||||
className="w-full text-left rounded-lg border border-gray-200 bg-white px-5 py-4 transition hover:border-gray-300 hover:bg-gray-50"
|
className="w-full text-left rounded-lg border border-gray-200 bg-white px-5 py-4 transition hover:border-gray-300 hover:bg-gray-50"
|
||||||
>
|
>
|
||||||
<span className={`block leading-snug ${isFocused ? "text-sm font-medium text-gray-900" : "text-sm text-gray-600"}`}>{node.label}</span>
|
<span className={`block leading-snug ${isFocused ? "text-sm font-medium text-gray-900" : "text-sm text-gray-600"}`}>{openQuestion}</span>
|
||||||
{node.description && node.description !== node.label && (
|
{node.description && node.description !== openQuestion && (
|
||||||
<p className="mt-1.5 text-xs leading-snug text-gray-500">{node.description}</p>
|
<p className="mt-1.5 text-xs leading-snug text-gray-500">{node.description}</p>
|
||||||
)}
|
)}
|
||||||
{!isFocused && <span className="mt-2 block text-[10px] uppercase tracking-wider text-gray-400">Unclear</span>}
|
{!isFocused && <span className="mt-2 block text-[10px] uppercase tracking-wider text-gray-400">Unclear</span>}
|
||||||
|
|||||||
+12
-1
@@ -12,9 +12,12 @@ import {
|
|||||||
} from "../lib/reconstruction/prompt.js";
|
} from "../lib/reconstruction/prompt.js";
|
||||||
import { normaliseAnalysisResponse } from "../lib/reconstruction/compatibility.js";
|
import { normaliseAnalysisResponse } from "../lib/reconstruction/compatibility.js";
|
||||||
import {
|
import {
|
||||||
|
reconstructionGenerationJsonSchema,
|
||||||
|
reconstructionGenerationSchema,
|
||||||
reconstructionV2Schema,
|
reconstructionV2Schema,
|
||||||
reconstructionSchema as reconstructionV1Schema,
|
reconstructionSchema as reconstructionV1Schema,
|
||||||
} from "../lib/reconstruction/schema.js";
|
} from "../lib/reconstruction/schema.js";
|
||||||
|
import { validateReconstructionOpenQuestions } from "../lib/reconstruction/open-question-presentation.js";
|
||||||
|
|
||||||
const MAX_SCENARIO_LENGTH = 10000;
|
const MAX_SCENARIO_LENGTH = 10000;
|
||||||
|
|
||||||
@@ -80,6 +83,8 @@ export async function analyseScenario(scenario, opts = {}) {
|
|||||||
const providerResult = await provider.generateReconstruction(
|
const providerResult = await provider.generateReconstruction(
|
||||||
promptObj.prompt,
|
promptObj.prompt,
|
||||||
reconstructionModelName,
|
reconstructionModelName,
|
||||||
|
reconstructionGenerationJsonSchema,
|
||||||
|
{ outputZodSchema: reconstructionGenerationSchema },
|
||||||
);
|
);
|
||||||
if (
|
if (
|
||||||
providerResult &&
|
providerResult &&
|
||||||
@@ -201,6 +206,9 @@ function buildCompatibilityDiagnostics(compatibility) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function buildSuccessResultV2(data, model, duration, version, compatibility) {
|
function buildSuccessResultV2(data, model, duration, version, compatibility) {
|
||||||
|
const presentationValidation = validateReconstructionOpenQuestions(
|
||||||
|
data.reconstruction,
|
||||||
|
);
|
||||||
return {
|
return {
|
||||||
success: true,
|
success: true,
|
||||||
validationStatus: "valid",
|
validationStatus: "valid",
|
||||||
@@ -209,10 +217,13 @@ function buildSuccessResultV2(data, model, duration, version, compatibility) {
|
|||||||
rawResponse: JSON.stringify(data).slice(0, 3000),
|
rawResponse: JSON.stringify(data).slice(0, 3000),
|
||||||
promptVersion: version,
|
promptVersion: version,
|
||||||
inputClassification: data.inputClassification,
|
inputClassification: data.inputClassification,
|
||||||
reconstruction: data.reconstruction,
|
reconstruction: presentationValidation.reconstruction,
|
||||||
evidence: data.evidence,
|
evidence: data.evidence,
|
||||||
nextQuestion: data.nextQuestion,
|
nextQuestion: data.nextQuestion,
|
||||||
errors: undefined,
|
errors: undefined,
|
||||||
|
presentationValidationStatus:
|
||||||
|
presentationValidation.issues.length > 0 ? "partial" : "valid",
|
||||||
|
presentationValidationIssues: presentationValidation.issues,
|
||||||
...buildCompatibilityDiagnostics(compatibility),
|
...buildCompatibilityDiagnostics(compatibility),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -184,6 +184,7 @@ export function buildInitialGraph(analysisData) {
|
|||||||
null,
|
null,
|
||||||
unk.confidence || "low",
|
unk.confidence || "low",
|
||||||
);
|
);
|
||||||
|
if (unk.question) node.openQuestion = unk.question;
|
||||||
mapSourceId(unk.id, node);
|
mapSourceId(unk.id, node);
|
||||||
unknownNodes.push(node);
|
unknownNodes.push(node);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -88,6 +88,10 @@ function buildDiagnostics({
|
|||||||
compatibilityApplied: analysis?.compatibilityApplied ?? false,
|
compatibilityApplied: analysis?.compatibilityApplied ?? false,
|
||||||
compatibilityChanges: analysis?.compatibilityChanges ?? [],
|
compatibilityChanges: analysis?.compatibilityChanges ?? [],
|
||||||
compatibilityWarnings: analysis?.compatibilityWarnings ?? [],
|
compatibilityWarnings: analysis?.compatibilityWarnings ?? [],
|
||||||
|
presentationValidationStatus:
|
||||||
|
analysis?.presentationValidationStatus ?? "not_applicable",
|
||||||
|
presentationValidationIssues:
|
||||||
|
analysis?.presentationValidationIssues ?? [],
|
||||||
unknownSelectionExplanation: unknownSelectionExplanation ?? null,
|
unknownSelectionExplanation: unknownSelectionExplanation ?? null,
|
||||||
reconstructionQuestion: reconstructionQuestion ?? null,
|
reconstructionQuestion: reconstructionQuestion ?? null,
|
||||||
reconstructionQuestionAccepted: reconstructionQuestionAccepted ?? null,
|
reconstructionQuestionAccepted: reconstructionQuestionAccepted ?? null,
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ export const situationNodeSchema = z.object({
|
|||||||
id: z.string().min(1),
|
id: z.string().min(1),
|
||||||
label: z.string().min(1),
|
label: z.string().min(1),
|
||||||
description: z.string().min(1),
|
description: z.string().min(1),
|
||||||
|
openQuestion: z.string().min(1).optional(),
|
||||||
kind: z.enum(Object.values(SituationKind)),
|
kind: z.enum(Object.values(SituationKind)),
|
||||||
status: z.enum(Object.values(SituationStatus)),
|
status: z.enum(Object.values(SituationStatus)),
|
||||||
confidence: z.enum(Object.values(ConfidenceLevel)),
|
confidence: z.enum(Object.values(ConfidenceLevel)),
|
||||||
@@ -259,6 +260,7 @@ export function makeNode(opts) {
|
|||||||
id,
|
id,
|
||||||
label: opts.label,
|
label: opts.label,
|
||||||
description: opts.description ?? opts.label,
|
description: opts.description ?? opts.label,
|
||||||
|
openQuestion: opts.openQuestion,
|
||||||
kind: opts.kind ?? "observation",
|
kind: opts.kind ?? "observation",
|
||||||
status: opts.status ?? "unknown",
|
status: opts.status ?? "unknown",
|
||||||
confidence: opts.confidence ?? "medium",
|
confidence: opts.confidence ?? "medium",
|
||||||
|
|||||||
+17
-5
@@ -546,10 +546,15 @@ class OpenAIReconstructionProvider {
|
|||||||
this.fetchImpl = fetchImpl;
|
this.fetchImpl = fetchImpl;
|
||||||
}
|
}
|
||||||
|
|
||||||
async generateReconstruction(prompt, modelName = "gpt-5.6-terra", outputSchema, { signal } = {}) {
|
async generateReconstruction(
|
||||||
|
prompt,
|
||||||
|
modelName = "gpt-5.6-terra",
|
||||||
|
outputSchema,
|
||||||
|
{ signal, outputZodSchema } = {},
|
||||||
|
) {
|
||||||
if (!this.apiKey) throw new Error("OPENAI_API_KEY is not set");
|
if (!this.apiKey) throw new Error("OPENAI_API_KEY is not set");
|
||||||
const structuredOutputSchema = outputSchema
|
const structuredOutputSchema = outputSchema
|
||||||
? createOpenAIStrictSchema(outputSchema, null)
|
? createOpenAIStrictSchema(outputSchema, outputZodSchema ?? null)
|
||||||
: openAIReconstructionJsonSchema;
|
: openAIReconstructionJsonSchema;
|
||||||
|
|
||||||
const format = {
|
const format = {
|
||||||
@@ -594,10 +599,17 @@ class OpenAIReconstructionProvider {
|
|||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const recoveredResponse = recoverJson(outputText);
|
||||||
|
const normalisedResponse = outputSchema && !outputZodSchema
|
||||||
|
? recoveredResponse
|
||||||
|
: normaliseOpenAITransportResponse(
|
||||||
|
recoveredResponse,
|
||||||
|
outputZodSchema ?? reconstructionV2Schema,
|
||||||
|
outputSchema ?? reconstructionJsonSchema,
|
||||||
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
response: outputSchema
|
response: normalisedResponse,
|
||||||
? recoverJson(outputText)
|
|
||||||
: normaliseOpenAITransportResponse(recoverJson(outputText)),
|
|
||||||
providerApiPath: "/v1/responses",
|
providerApiPath: "/v1/responses",
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -274,8 +274,14 @@ export function buildFacilitatorViewModel(_ref) {
|
|||||||
// Scaffold items are entirely suppressed from user-facing sections.
|
// Scaffold items are entirely suppressed from user-facing sections.
|
||||||
if (semanticRole === "scaffolding") continue;
|
if (semanticRole === "scaffolding") continue;
|
||||||
|
|
||||||
|
// An unknown's openQuestion is a model-authored presentation projection.
|
||||||
|
// Its label and description remain the canonical uncertainty representation.
|
||||||
|
var displayText =
|
||||||
|
node.kind === "unknown" && !resolved && node.openQuestion
|
||||||
|
? node.openQuestion.trim()
|
||||||
|
: displayResult.displayText;
|
||||||
var entry = {
|
var entry = {
|
||||||
text: displayResult.displayText,
|
text: displayText,
|
||||||
normalised: normaliseText(displayResult.displayText),
|
normalised: normaliseText(displayResult.displayText),
|
||||||
kind: node.kind,
|
kind: node.kind,
|
||||||
semanticRole: semanticRole,
|
semanticRole: semanticRole,
|
||||||
@@ -410,8 +416,10 @@ export function buildFacilitatorViewModel(_ref) {
|
|||||||
if (!a.isActiveUnknown && b.isActiveUnknown) return 1;
|
if (!a.isActiveUnknown && b.isActiveUnknown) return 1;
|
||||||
|
|
||||||
// Selected-question target: match by normalised text
|
// Selected-question target: match by normalised text
|
||||||
if (sqText && a.normalised === sqText && b.normalised !== sqText) return -1;
|
var aMatchesSelectedQuestion = sqText && normaliseText(a.text) === sqText;
|
||||||
if (sqText && a.normalised !== sqText && b.normalised === sqText) return 1;
|
var bMatchesSelectedQuestion = sqText && normaliseText(b.text) === sqText;
|
||||||
|
if (aMatchesSelectedQuestion && !bMatchesSelectedQuestion) return -1;
|
||||||
|
if (!aMatchesSelectedQuestion && bMatchesSelectedQuestion) return 1;
|
||||||
|
|
||||||
// Explicit priority fields where available in the graph
|
// Explicit priority fields where available in the graph
|
||||||
var pa = a.priority != null ? a.priority : null;
|
var pa = a.priority != null ? a.priority : null;
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
const ISSUE_MESSAGES = {
|
||||||
|
question_missing: "No user-facing Open Question was generated.",
|
||||||
|
question_empty: "The generated Open Question was empty.",
|
||||||
|
question_not_interrogative: "The generated Open Question was not a direct interrogative.",
|
||||||
|
question_indirect_proposition:
|
||||||
|
"The generated Open Question used an indirect uncertainty proposition.",
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Narrow presentation validation only. This does not attempt to prove semantic
|
||||||
|
* equivalence with the canonical uncertainty and never rewrites either value.
|
||||||
|
*/
|
||||||
|
export function validateOpenQuestionCandidate(candidate) {
|
||||||
|
if (candidate === undefined || candidate === null) {
|
||||||
|
return { valid: false, category: "question_missing" };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof candidate !== "string" || candidate.trim().length === 0) {
|
||||||
|
return { valid: false, category: "question_empty" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const question = candidate.trim();
|
||||||
|
if (/^whether\b/i.test(question)) {
|
||||||
|
return { valid: false, category: "question_indirect_proposition" };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!question.endsWith("?")) {
|
||||||
|
return { valid: false, category: "question_not_interrogative" };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { valid: true, question };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Remove invalid presentation candidates without changing canonical descriptions,
|
||||||
|
* IDs, ordering, relationships, or any other reconstruction content.
|
||||||
|
*/
|
||||||
|
export function validateReconstructionOpenQuestions(reconstruction) {
|
||||||
|
const issues = [];
|
||||||
|
const importantUnknowns = (reconstruction?.importantUnknowns ?? []).map(
|
||||||
|
(unknown, index) => {
|
||||||
|
const validation = validateOpenQuestionCandidate(unknown.question);
|
||||||
|
if (validation.valid) {
|
||||||
|
return { ...unknown, question: validation.question };
|
||||||
|
}
|
||||||
|
|
||||||
|
const { question: _invalidQuestion, ...canonicalUnknown } = unknown;
|
||||||
|
issues.push({
|
||||||
|
category: validation.category,
|
||||||
|
path: ["reconstruction", "importantUnknowns", index, "question"],
|
||||||
|
unknownId: unknown.id,
|
||||||
|
message: ISSUE_MESSAGES[validation.category],
|
||||||
|
severity: "nonfatal",
|
||||||
|
});
|
||||||
|
return canonicalUnknown;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
reconstruction: { ...reconstruction, importantUnknowns },
|
||||||
|
issues,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -144,6 +144,21 @@ const reconstructionRelationshipSchema = z.object({
|
|||||||
confidence: confidenceEnum,
|
confidence: confidenceEnum,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const importantUnknownSchema = itemSchemaV1.extend({
|
||||||
|
// Canonical uncertainty meaning remains in description. question is an
|
||||||
|
// optional, model-authored user-facing interrogative for the Open Questions UI.
|
||||||
|
// Keep this tolerant for historical data and nonfatal presentation validation.
|
||||||
|
question: z.string().optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const generatedImportantUnknownSchema = itemSchemaV1.extend({
|
||||||
|
// New generation requests require a distinct presentation candidate. More
|
||||||
|
// nuanced interrogative validation is intentionally performed locally after
|
||||||
|
// canonical reconstruction validation rather than encoded as provider-specific
|
||||||
|
// JSON Schema grammar.
|
||||||
|
question: z.string().trim().min(1),
|
||||||
|
});
|
||||||
|
|
||||||
const reconstructionSchemaV2 = z.object({
|
const reconstructionSchemaV2 = z.object({
|
||||||
summary: z.string().min(1),
|
summary: z.string().min(1),
|
||||||
actors: z.array(itemSchemaV1),
|
actors: z.array(itemSchemaV1),
|
||||||
@@ -167,7 +182,7 @@ const reconstructionSchemaV2 = z.object({
|
|||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
contradictions: z.array(itemSchemaV1),
|
contradictions: z.array(itemSchemaV1),
|
||||||
importantUnknowns: z.array(itemSchemaV1),
|
importantUnknowns: z.array(importantUnknownSchema),
|
||||||
plausibleInterpretations: z.array(
|
plausibleInterpretations: z.array(
|
||||||
z.object({
|
z.object({
|
||||||
id: z.string().min(1),
|
id: z.string().min(1),
|
||||||
@@ -205,6 +220,19 @@ export const reconstructionV2Schema = z.object({
|
|||||||
nextQuestion: nextQuestionSchema,
|
nextQuestion: nextQuestionSchema,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Generation-only contract. Durable reconstruction parsing remains tolerant so
|
||||||
|
// historical snapshots and otherwise valid canonical reconstructions do not fail
|
||||||
|
// solely because presentation wording is absent or malformed.
|
||||||
|
export const reconstructionGenerationSchema = reconstructionV2Schema.extend({
|
||||||
|
reconstruction: reconstructionSchemaV2.extend({
|
||||||
|
importantUnknowns: z.array(generatedImportantUnknownSchema),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const reconstructionGenerationJsonSchema = z.toJSONSchema(
|
||||||
|
reconstructionGenerationSchema,
|
||||||
|
);
|
||||||
|
|
||||||
// Outer wrapper for API return (includes diagnostics + v0.2 data)
|
// Outer wrapper for API return (includes diagnostics + v0.2 data)
|
||||||
export const analyseResponseV2Schema = z.object({
|
export const analyseResponseV2Schema = z.object({
|
||||||
inputClassification: inputClassificationSchema.optional(),
|
inputClassification: inputClassificationSchema.optional(),
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ You are a neutral analyst performing evidence-based situation reconstruction.
|
|||||||
7. Distinguish: what was said / what it may mean / why it may have been said.
|
7. Distinguish: what was said / what it may mean / why it may have been said.
|
||||||
8. If input is too ambiguous or contains no useful operational anchors, say so and ask for
|
8. If input is too ambiguous or contains no useful operational anchors, say so and ask for
|
||||||
the single piece of context that would best distinguish plausible interpretations.
|
the single piece of context that would best distinguish plausible interpretations.
|
||||||
|
9. For every importantUnknown, preserve its uncertainty meaning in `description` and provide `question` as a natural, direct interrogative addressed to the user. Retain all scope, conditions, timeframes, and qualifications without adding assumptions or an implied conclusion. Do not use fragments or indirect proposition forms such as "Whether …".
|
||||||
|
|
||||||
## Confidence scale
|
## Confidence scale
|
||||||
|
|
||||||
@@ -83,7 +84,7 @@ Return a JSON object with exactly these four top-level keys (use **camelCase**):
|
|||||||
"knownTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}],
|
"knownTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}],
|
||||||
"unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "..."}],
|
"unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "..."}],
|
||||||
"contradictions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
"contradictions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||||
"importantUnknowns": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
"importantUnknowns": [{"id": "...", "description": "<the canonical uncertainty meaning>", "question": "<a natural direct question for the user>", "confidence": "<low|medium|high>"}],
|
||||||
"plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": ["<ids that support this interpretation>"], "assumptionsRequired": [], "confidence": "<low|medium|high>"}]
|
"plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": ["<ids that support this interpretation>"], "assumptionsRequired": [], "confidence": "<low|medium|high>"}]
|
||||||
},
|
},
|
||||||
"evidence": [
|
"evidence": [
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ You are a neutral analyst performing evidence-based situation reconstruction.
|
|||||||
8. Distinguish: what was said / what it may mean / why it may have been said.
|
8. Distinguish: what was said / what it may mean / why it may have been said.
|
||||||
9. If input is too ambiguous or contains no useful operational anchors, say so and ask for
|
9. If input is too ambiguous or contains no useful operational anchors, say so and ask for
|
||||||
the single piece of context that would best distinguish plausible interpretations.
|
the single piece of context that would best distinguish plausible interpretations.
|
||||||
|
9a. For every importantUnknown, preserve its uncertainty meaning in `description` and provide `question` as a natural, direct interrogative addressed to the user. Retain all scope, conditions, timeframes, and qualifications without adding assumptions or an implied conclusion. Do not use fragments or indirect proposition forms such as "Whether …".
|
||||||
|
|
||||||
10. Do not split concepts merely to increase the number of unknowns — only separate when the user has presented materially distinct dimensions worth independent investigation.
|
10. Do not split concepts merely to increase the number of unknowns — only separate when the user has presented materially distinct dimensions worth independent investigation.
|
||||||
|
|
||||||
@@ -123,7 +124,7 @@ Return a JSON object with exactly these four top-level keys (use **camelCase**):
|
|||||||
"knownTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}],
|
"knownTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}],
|
||||||
"unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "..."}],
|
"unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "..."}],
|
||||||
"contradictions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
"contradictions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||||
"importantUnknowns": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
"importantUnknowns": [{"id": "...", "description": "<the canonical uncertainty meaning>", "question": "<a natural direct question for the user>", "confidence": "<low|medium|high>"}],
|
||||||
"plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": ["<ids that support this interpretation>"], "assumptionsRequired": [], "confidence": "<low|medium|high>"}]
|
"plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": ["<ids that support this interpretation>"], "assumptionsRequired": [], "confidence": "<low|medium|high>"}]
|
||||||
},
|
},
|
||||||
"evidence": [
|
"evidence": [
|
||||||
|
|||||||
@@ -29,6 +29,8 @@ You are a neutral analyst performing evidence-based situation reconstruction.
|
|||||||
9. If input is too ambiguous or contains no useful operational anchors, say so and ask for
|
9. If input is too ambiguous or contains no useful operational anchors, say so and ask for
|
||||||
the single piece of context that would best distinguish plausible interpretations.
|
the single piece of context that would best distinguish plausible interpretations.
|
||||||
|
|
||||||
|
9a. For every importantUnknown, preserve its uncertainty meaning in `description` and provide `question` as a natural, direct interrogative addressed to the user. Retain all scope, conditions, timeframes, and qualifications without adding assumptions or an implied conclusion. Do not use fragments or indirect proposition forms such as "Whether …".
|
||||||
|
|
||||||
10. Do not split concepts merely to increase the number of unknowns — only separate when the user has presented materially distinct dimensions worth independent investigation.
|
10. Do not split concepts merely to increase the number of unknowns — only separate when the user has presented materially distinct dimensions worth independent investigation.
|
||||||
|
|
||||||
## Preserve supplied relationships
|
## Preserve supplied relationships
|
||||||
@@ -139,7 +141,7 @@ Return a JSON object with exactly these four top-level keys (use **camelCase**):
|
|||||||
"knownTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}],
|
"knownTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}],
|
||||||
"unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "..."}],
|
"unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "..."}],
|
||||||
"contradictions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
"contradictions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||||
"importantUnknowns": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
"importantUnknowns": [{"id": "...", "description": "<the canonical uncertainty meaning>", "question": "<a natural direct question for the user>", "confidence": "<low|medium|high>"}],
|
||||||
"plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": ["<ids that support this interpretation>"], "assumptionsRequired": [], "confidence": "<low|medium|high>"}]
|
"plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": ["<ids that support this interpretation>"], "assumptionsRequired": [], "confidence": "<low|medium|high>"}]
|
||||||
},
|
},
|
||||||
"evidence": [
|
"evidence": [
|
||||||
|
|||||||
@@ -26,6 +26,8 @@ You are a neutral analyst performing evidence-based situation reconstruction.
|
|||||||
|
|
||||||
7. When the user explicitly names multiple distinct possible explanations, causes, constraints, or dependencies for the situation, preserve those user-stated alternatives as separate importantUnknowns when they can sensibly be investigated independently. Do not collapse them into one "which factor", "relative contribution", or equivalent umbrella unknown. Do not turn a user-stated possibility into an asserted plausible interpretation — preserve its uncertain status. Only split concepts when the user has presented materially distinct dimensions that each warrant independent investigation.
|
7. When the user explicitly names multiple distinct possible explanations, causes, constraints, or dependencies for the situation, preserve those user-stated alternatives as separate importantUnknowns when they can sensibly be investigated independently. Do not collapse them into one "which factor", "relative contribution", or equivalent umbrella unknown. Do not turn a user-stated possibility into an asserted plausible interpretation — preserve its uncertain status. Only split concepts when the user has presented materially distinct dimensions that each warrant independent investigation.
|
||||||
|
|
||||||
|
7a. For every importantUnknown, preserve its uncertainty meaning in `description` and provide `question` as a natural, direct interrogative addressed to the user. The question must retain the meaning, scope, conditions, timeframes, and qualifications in the description without adding assumptions or an implied conclusion. Do not use fragments or indirect proposition forms such as "Whether …". This applies across domains.
|
||||||
|
|
||||||
8. Distinguish: what was said / what it may mean / why it may have been said.
|
8. Distinguish: what was said / what it may mean / why it may have been said.
|
||||||
|
|
||||||
9. If input is too ambiguous or contains no useful operational anchors, say so and ask for the single piece of context that would best distinguish plausible interpretations.
|
9. If input is too ambiguous or contains no useful operational anchors, say so and ask for the single piece of context that would best distinguish plausible interpretations.
|
||||||
@@ -152,7 +154,7 @@ Return a JSON object with exactly these four top-level keys (use **camelCase**):
|
|||||||
"knownTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}],
|
"knownTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}],
|
||||||
"unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "..."}],
|
"unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "..."}],
|
||||||
"contradictions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
"contradictions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||||
"importantUnknowns": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
"importantUnknowns": [{"id": "...", "description": "<the canonical uncertainty meaning>", "question": "<a natural direct question for the user>", "confidence": "<low|medium|high>"}],
|
||||||
"plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": ["<ids that support this interpretation>"], "assumptionsRequired": [], "confidence": "<low|medium|high>"}],
|
"plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": ["<ids that support this interpretation>"], "assumptionsRequired": [], "confidence": "<low|medium|high>"}],
|
||||||
"relationships": [{"id": "...", "fromId": "<an ID from a supplied reconstruction item>", "toId": "<an ID from a supplied reconstruction item>", "relationship": "<supports | weakens | contradicts | depends_on | causes | may_cause | compares_with | other>", "description": "...", "confidence": "<low|medium|high>"}]
|
"relationships": [{"id": "...", "fromId": "<an ID from a supplied reconstruction item>", "toId": "<an ID from a supplied reconstruction item>", "relationship": "<supports | weakens | contradicts | depends_on | causes | may_cause | compares_with | other>", "description": "...", "confidence": "<low|medium|high>"}]
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -7,11 +7,13 @@ function mkN(id, label, opts) {
|
|||||||
var k = (opts && opts.kind) || "unknown";
|
var k = (opts && opts.kind) || "unknown";
|
||||||
var s = (opts && opts.status) || (k === "unknown" ? "unknown" : "known");
|
var s = (opts && opts.status) || (k === "unknown" ? "unknown" : "known");
|
||||||
var c = (opts && opts.confidence) || "low";
|
var c = (opts && opts.confidence) || "low";
|
||||||
return {
|
var node = {
|
||||||
id, label, description: (opts && opts.description) || label, kind: k, status: s, confidence: c,
|
id, label, description: (opts && opts.description) || label, kind: k, status: s, confidence: c,
|
||||||
confidenceAssessment: { evidenceConfidence: c, completenessStatus: "partial", conclusionConfidence: c },
|
confidenceAssessment: { evidenceConfidence: c, completenessStatus: "partial", conclusionConfidence: c },
|
||||||
value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], childIds: []
|
value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], childIds: []
|
||||||
};
|
};
|
||||||
|
if (opts && opts.openQuestion) node.openQuestion = opts.openQuestion;
|
||||||
|
return node;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ── Filtering tests ─────────────────────────────────────────────── */
|
/* ── Filtering tests ─────────────────────────────────────────────── */
|
||||||
@@ -147,6 +149,41 @@ describe("Deduplication", () => {
|
|||||||
});
|
});
|
||||||
// The unresolved one (if any) or the cross-dedup should prevent duplication
|
// The unresolved one (if any) or the cross-dedup should prevent duplication
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("keeps canonically distinct uncertainties with identical presentation questions", () => {
|
||||||
|
const sharedQuestion = "What evidence would resolve this uncertainty?";
|
||||||
|
const vm = buildFacilitatorViewModel({
|
||||||
|
nodes: [
|
||||||
|
mkN("u1", "Whether supplier capacity is sufficient for launch", {
|
||||||
|
openQuestion: sharedQuestion,
|
||||||
|
}),
|
||||||
|
mkN("u2", "Whether customer demand is sufficient for launch", {
|
||||||
|
openQuestion: sharedQuestion,
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
resolvedIds: new Set(),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(vm.investigating.items.filter((item) => item === sharedQuestion)).toHaveLength(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("continues to merge genuine canonical duplicates despite different presentation questions", () => {
|
||||||
|
const vm = buildFacilitatorViewModel({
|
||||||
|
nodes: [
|
||||||
|
mkN("u1", "Whether supplier capacity is sufficient for launch", {
|
||||||
|
openQuestion: "Can the supplier support the launch volume?",
|
||||||
|
}),
|
||||||
|
mkN("u2", "whether supplier capacity is sufficient for launch!", {
|
||||||
|
openQuestion: "Is enough supplier capacity available?",
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
resolvedIds: new Set(),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(vm.investigating.items).toEqual([
|
||||||
|
"Can the supplier support the launch volume?",
|
||||||
|
]);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
/* ── Ranking tests ───────────────────────────────────────────────── */
|
/* ── Ranking tests ───────────────────────────────────────────────── */
|
||||||
|
|||||||
@@ -0,0 +1,204 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { buildInitialGraph } from "@/lib/graph/builder.js";
|
||||||
|
import { formulateQuestionForTarget } from "@/lib/graph/focused-investigation.js";
|
||||||
|
import { makeGraph, makeNodeId } from "@/lib/graph/schema.js";
|
||||||
|
import { buildFacilitatorViewModel } from "@/lib/presentation/facilitator-view-adapter.js";
|
||||||
|
import { buildPrompt } from "@/lib/reconstruction/prompt.js";
|
||||||
|
import { validateReconstructionOpenQuestions } from "@/lib/reconstruction/open-question-presentation.js";
|
||||||
|
import { reconstructionV2Schema } from "@/lib/reconstruction/schema.js";
|
||||||
|
|
||||||
|
function reconstructionWithUnknowns(importantUnknowns) {
|
||||||
|
return {
|
||||||
|
inputClassification: {
|
||||||
|
primaryType: "other",
|
||||||
|
secondaryTypes: [],
|
||||||
|
reasoningModes: ["identify_missing_information"],
|
||||||
|
classificationReason: "The scenario contains unresolved factors.",
|
||||||
|
confidence: "medium",
|
||||||
|
},
|
||||||
|
reconstruction: {
|
||||||
|
summary: "A situation with unresolved factors.",
|
||||||
|
actors: [],
|
||||||
|
systemsOrObjects: [],
|
||||||
|
expectedStates: [],
|
||||||
|
observedStates: [],
|
||||||
|
differences: [],
|
||||||
|
knownTransitions: [],
|
||||||
|
unexplainedTransitions: [],
|
||||||
|
contradictions: [],
|
||||||
|
importantUnknowns,
|
||||||
|
plausibleInterpretations: [],
|
||||||
|
relationships: [],
|
||||||
|
},
|
||||||
|
evidence: [],
|
||||||
|
nextQuestion: {
|
||||||
|
id: "next-question",
|
||||||
|
question: "What information would clarify the situation?",
|
||||||
|
targets: [],
|
||||||
|
reason: "The situation has unresolved factors.",
|
||||||
|
expectedInformationValue: "medium",
|
||||||
|
reasoningMode: "identify_missing_information",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const CASES = [
|
||||||
|
{
|
||||||
|
name: "recruitment uncertainty",
|
||||||
|
description: "Whether skilled production staff can be recruited within the required timeframe",
|
||||||
|
question: "Can skilled production staff be recruited within the required timeframe?",
|
||||||
|
terms: ["skilled production staff", "required timeframe"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "demand uncertainty",
|
||||||
|
description: "The level of demand in the North West market needed to justify the expansion cost",
|
||||||
|
question: "What level of demand in the North West market is needed to justify the expansion cost?",
|
||||||
|
terms: ["North West market", "justify the expansion cost"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "historical uncertainty",
|
||||||
|
description: "Whether the prior staffing level was sufficient before the expansion",
|
||||||
|
question: "Was the prior staffing level sufficient before the expansion?",
|
||||||
|
terms: ["prior staffing level", "before the expansion"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "non-business uncertainty",
|
||||||
|
description: "The time at which the missing hiker last used the eastern trail",
|
||||||
|
question: "When did the missing hiker last use the eastern trail?",
|
||||||
|
terms: ["missing hiker", "eastern trail"],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
describe("Open Question presentation", () => {
|
||||||
|
it.each(["v0.2", "v0.3", "v0.4", "v0.5"])("instructs reconstruction prompt %s to produce a direct Open Question", async (version) => {
|
||||||
|
const { prompt } = await buildPrompt("A scenario with an unresolved condition.", version);
|
||||||
|
|
||||||
|
expect(prompt).toContain("importantUnknown");
|
||||||
|
expect(prompt).toContain("natural, direct interrogative");
|
||||||
|
expect(prompt).toContain('"question"');
|
||||||
|
expect(prompt).toContain('"Whether …"');
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each(CASES)("preserves $name meaning while displaying a direct interrogative", ({ description, question, terms }) => {
|
||||||
|
const parsed = reconstructionV2Schema.parse(
|
||||||
|
reconstructionWithUnknowns([{ id: "unknown", description, question, confidence: "medium" }]),
|
||||||
|
);
|
||||||
|
const graph = buildInitialGraph({ reconstruction: parsed.reconstruction, evidence: [] });
|
||||||
|
const unknown = graph.nodes.find((node) => node.kind === "unknown");
|
||||||
|
const viewModel = buildFacilitatorViewModel({ nodes: graph.nodes, resolvedIds: new Set() });
|
||||||
|
|
||||||
|
expect(unknown.label).toBe(description);
|
||||||
|
expect(unknown.description).toBe(description);
|
||||||
|
expect(unknown.openQuestion).toBe(question);
|
||||||
|
expect(viewModel.investigating.items).toContain(question);
|
||||||
|
expect(question).toMatch(/\?$/);
|
||||||
|
expect(question).not.toMatch(/^whether\b/i);
|
||||||
|
expect(question).not.toMatch(/^before whether\b/i);
|
||||||
|
for (const term of terms) expect(question.toLowerCase()).toContain(term.toLowerCase());
|
||||||
|
});
|
||||||
|
|
||||||
|
it("leaves canonical identity, explicit target selection, and focused formulation unchanged", () => {
|
||||||
|
const description = CASES[0].description;
|
||||||
|
const question = CASES[0].question;
|
||||||
|
const graphData = buildInitialGraph({
|
||||||
|
reconstruction: reconstructionWithUnknowns([
|
||||||
|
{ id: "recruitment", description, question, confidence: "medium" },
|
||||||
|
]).reconstruction,
|
||||||
|
evidence: [],
|
||||||
|
});
|
||||||
|
const target = graphData.nodes.find((node) => node.kind === "unknown");
|
||||||
|
const graph = makeGraph({
|
||||||
|
centralStatement: "A manufacturer is considering expansion.",
|
||||||
|
nodes: graphData.nodes,
|
||||||
|
edges: graphData.edges,
|
||||||
|
activeUnknownNodeId: null,
|
||||||
|
resolvedNodeIds: [],
|
||||||
|
currentSummary: "Expansion depends on recruitment capacity.",
|
||||||
|
});
|
||||||
|
const before = JSON.stringify(graph);
|
||||||
|
const result = formulateQuestionForTarget({ situationGraph: graph, targetNodeId: target.id });
|
||||||
|
|
||||||
|
expect(target.id).toBe(makeNodeId(description));
|
||||||
|
expect(target.id).not.toBe(makeNodeId(question));
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
expect(result.targetNodeId).toBe(target.id);
|
||||||
|
expect(result.question).toMatch(/\?$/);
|
||||||
|
expect(result.question.toLowerCase()).toContain("staff");
|
||||||
|
expect(JSON.stringify(graph)).toBe(before);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps persisted legacy unknowns valid and falls back to canonical text when no display question exists", () => {
|
||||||
|
const legacyDescription = "Whether the figures are comparable";
|
||||||
|
const legacyNode = makeGraph({
|
||||||
|
centralStatement: "Compare two figures.",
|
||||||
|
nodes: [
|
||||||
|
{
|
||||||
|
id: makeNodeId(legacyDescription),
|
||||||
|
label: legacyDescription,
|
||||||
|
description: legacyDescription,
|
||||||
|
kind: "unknown",
|
||||||
|
status: "unknown",
|
||||||
|
confidence: "medium",
|
||||||
|
evidenceIds: [],
|
||||||
|
dependsOn: [],
|
||||||
|
affects: [],
|
||||||
|
parentId: null,
|
||||||
|
childIds: [],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
edges: [],
|
||||||
|
activeUnknownNodeId: null,
|
||||||
|
resolvedNodeIds: [],
|
||||||
|
currentSummary: "The comparison remains unresolved.",
|
||||||
|
});
|
||||||
|
const viewModel = buildFacilitatorViewModel({ nodes: legacyNode.nodes, resolvedIds: new Set() });
|
||||||
|
|
||||||
|
expect(viewModel.investigating.items).toContain(legacyDescription);
|
||||||
|
expect(legacyNode.nodes[0]).not.toHaveProperty("openQuestion");
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
["missing", undefined, false],
|
||||||
|
["empty", " ", true],
|
||||||
|
["whether fragment", "Whether the facility can recruit workers in time", true],
|
||||||
|
["whether fragment with punctuation", "Whether the facility can recruit workers in time?", true],
|
||||||
|
["statement", "The facility can recruit workers in time.", true],
|
||||||
|
])(
|
||||||
|
"keeps an invalid %s presentation canonically selectable and investigable",
|
||||||
|
(_name, question, includeQuestion) => {
|
||||||
|
const description =
|
||||||
|
"Whether the facility can recruit skilled workers within the launch timeframe.";
|
||||||
|
const unknown = {
|
||||||
|
id: "unknown",
|
||||||
|
description,
|
||||||
|
confidence: "medium",
|
||||||
|
};
|
||||||
|
if (includeQuestion) unknown.question = question;
|
||||||
|
const reconstruction = reconstructionWithUnknowns([unknown]).reconstruction;
|
||||||
|
const validated = validateReconstructionOpenQuestions(reconstruction);
|
||||||
|
const graphParts = buildInitialGraph({
|
||||||
|
reconstruction: validated.reconstruction,
|
||||||
|
evidence: [],
|
||||||
|
});
|
||||||
|
const graph = makeGraph({
|
||||||
|
centralStatement: "The facility launch depends on workforce availability.",
|
||||||
|
nodes: graphParts.nodes,
|
||||||
|
edges: graphParts.edges,
|
||||||
|
currentSummary: "Workforce availability remains unresolved.",
|
||||||
|
});
|
||||||
|
const graphUnknown = graph.nodes.find((node) => node.kind === "unknown");
|
||||||
|
const focused = formulateQuestionForTarget({
|
||||||
|
situationGraph: graph,
|
||||||
|
targetNodeId: graphUnknown.id,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(validated.issues).toHaveLength(1);
|
||||||
|
expect(graphUnknown.label).toBe(description);
|
||||||
|
expect(graphUnknown.description).toBe(description);
|
||||||
|
expect(graphUnknown).not.toHaveProperty("openQuestion");
|
||||||
|
expect(graphUnknown.id).toBe(makeNodeId(description));
|
||||||
|
expect(focused.success).toBe(true);
|
||||||
|
expect(focused.targetNodeId).toBe(graphUnknown.id);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
@@ -55,6 +55,8 @@ function makeAnalysisResult(overrides = {}) {
|
|||||||
compatibilityApplied: false,
|
compatibilityApplied: false,
|
||||||
compatibilityChanges: [],
|
compatibilityChanges: [],
|
||||||
compatibilityWarnings: [],
|
compatibilityWarnings: [],
|
||||||
|
presentationValidationStatus: "valid",
|
||||||
|
presentationValidationIssues: [],
|
||||||
...overrides,
|
...overrides,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -399,9 +401,37 @@ describe("lib/graph/orchestrator startCase", () => {
|
|||||||
valid: true,
|
valid: true,
|
||||||
errors: [],
|
errors: [],
|
||||||
});
|
});
|
||||||
|
expect(result.diagnostics.presentationValidationStatus).toBe("valid");
|
||||||
|
expect(result.diagnostics.presentationValidationIssues).toEqual([]);
|
||||||
expect(downstreamProvider.generateReconstruction).not.toHaveBeenCalled();
|
expect(downstreamProvider.generateReconstruction).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("propagates nonfatal Open Question presentation issues through diagnostics", async () => {
|
||||||
|
const issue = {
|
||||||
|
category: "question_indirect_proposition",
|
||||||
|
path: ["reconstruction", "importantUnknowns", 0, "question"],
|
||||||
|
unknownId: "unk-1",
|
||||||
|
message: "The generated Open Question used an indirect uncertainty proposition.",
|
||||||
|
severity: "nonfatal",
|
||||||
|
};
|
||||||
|
mockAnalyseScenario.mockResolvedValue(
|
||||||
|
makeAnalysisResult({
|
||||||
|
presentationValidationStatus: "partial",
|
||||||
|
presentationValidationIssues: [issue],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const { startCase } = await import("@/lib/graph/orchestrator.js");
|
||||||
|
|
||||||
|
const result = await startCase(
|
||||||
|
{ scenario: "Scenario text" },
|
||||||
|
{ reconstructionOnly: true },
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
expect(result.diagnostics.presentationValidationStatus).toBe("partial");
|
||||||
|
expect(result.diagnostics.presentationValidationIssues).toEqual([issue]);
|
||||||
|
});
|
||||||
|
|
||||||
it("rejects invalid request input without throwing", async () => {
|
it("rejects invalid request input without throwing", async () => {
|
||||||
const { startCase } = await import("@/lib/graph/orchestrator.js");
|
const { startCase } = await import("@/lib/graph/orchestrator.js");
|
||||||
|
|
||||||
|
|||||||
+136
-1
@@ -8,7 +8,11 @@ import {
|
|||||||
normaliseOpenAITransportResponse,
|
normaliseOpenAITransportResponse,
|
||||||
reconstructionJsonSchema,
|
reconstructionJsonSchema,
|
||||||
} from "@/lib/llm/provider.js";
|
} from "@/lib/llm/provider.js";
|
||||||
import { reconstructionV2Schema } from "@/lib/reconstruction/schema.js";
|
import {
|
||||||
|
reconstructionGenerationJsonSchema,
|
||||||
|
reconstructionGenerationSchema,
|
||||||
|
reconstructionV2Schema,
|
||||||
|
} from "@/lib/reconstruction/schema.js";
|
||||||
import { focusedDeconstructJsonSchema } from "@/lib/graph/focused-investigation.js";
|
import { focusedDeconstructJsonSchema } from "@/lib/graph/focused-investigation.js";
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
|
|
||||||
@@ -299,6 +303,137 @@ describe("OpenAI reconstruction provider experiment seam", () => {
|
|||||||
expect(acceptsNull(transition.properties.id)).toBe(false);
|
expect(acceptsNull(transition.properties.id)).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("preserves optional transport nullability for the generation-specific schema", async () => {
|
||||||
|
const fetchImpl = vi.fn().mockResolvedValue({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({ output_text: "{}" }),
|
||||||
|
});
|
||||||
|
const provider = createOpenAIReconstructionProvider({
|
||||||
|
apiKey: "test-key",
|
||||||
|
fetchImpl,
|
||||||
|
});
|
||||||
|
|
||||||
|
await provider.generateReconstruction(
|
||||||
|
"prompt",
|
||||||
|
"test-model",
|
||||||
|
reconstructionGenerationJsonSchema,
|
||||||
|
{ outputZodSchema: reconstructionGenerationSchema },
|
||||||
|
);
|
||||||
|
|
||||||
|
const request = JSON.parse(fetchImpl.mock.calls[0][1].body);
|
||||||
|
const reconstruction = request.text.format.schema.properties.reconstruction;
|
||||||
|
const unknown = reconstruction.properties.importantUnknowns.items;
|
||||||
|
const unexplained = reconstruction.properties.unexplainedTransitions.items;
|
||||||
|
|
||||||
|
expect(unknown.required).toContain("question");
|
||||||
|
expect(unknown.properties.question.minLength).toBe(1);
|
||||||
|
expect(unexplained.required).toContain("entity");
|
||||||
|
expect(unexplained.properties.entity.anyOf).toContainEqual({ type: "null" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("normalises generation-schema null placeholders before canonical validation", async () => {
|
||||||
|
const transportResponse = {
|
||||||
|
inputClassification: {
|
||||||
|
primaryType: "unexplained_change",
|
||||||
|
secondaryTypes: null,
|
||||||
|
reasoningModes: null,
|
||||||
|
classificationReason: "A reported change requires reconstruction.",
|
||||||
|
confidence: "high",
|
||||||
|
},
|
||||||
|
reconstruction: {
|
||||||
|
summary: "A change occurred and its cause remains uncertain.",
|
||||||
|
actors: [],
|
||||||
|
systemsOrObjects: [],
|
||||||
|
expectedStates: [],
|
||||||
|
observedStates: [],
|
||||||
|
differences: [],
|
||||||
|
knownTransitions: [],
|
||||||
|
unexplainedTransitions: [
|
||||||
|
{
|
||||||
|
id: "transition-1",
|
||||||
|
description: "The service level declined after the release.",
|
||||||
|
confidence: "medium",
|
||||||
|
entity: null,
|
||||||
|
previousState: null,
|
||||||
|
currentState: null,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
contradictions: [],
|
||||||
|
importantUnknowns: [
|
||||||
|
{
|
||||||
|
id: "unknown-1",
|
||||||
|
description: "Whether the release caused the service-level decline",
|
||||||
|
question: "Did the release cause the service-level decline?",
|
||||||
|
confidence: "medium",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
plausibleInterpretations: [
|
||||||
|
{
|
||||||
|
id: "interpretation-1",
|
||||||
|
description: "The release may have reduced service capacity.",
|
||||||
|
supportingEvidenceIds: ["evidence-1"],
|
||||||
|
assumptionsRequired: null,
|
||||||
|
confidence: "low",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
relationships: null,
|
||||||
|
},
|
||||||
|
evidence: [
|
||||||
|
{
|
||||||
|
id: "evidence-1",
|
||||||
|
description: "The decline was reported after the release.",
|
||||||
|
evidenceType: "reported_statement",
|
||||||
|
source: null,
|
||||||
|
attribution: null,
|
||||||
|
confidence: "medium",
|
||||||
|
importance: "important",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
nextQuestion: {
|
||||||
|
id: "next-1",
|
||||||
|
question: "What changed in service capacity after the release?",
|
||||||
|
targets: ["unknown-1"],
|
||||||
|
reason: "The cause of the decline remains unresolved.",
|
||||||
|
expectedInformationValue: "high",
|
||||||
|
reasoningMode: null,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const fetchImpl = vi.fn().mockResolvedValue({
|
||||||
|
ok: true,
|
||||||
|
json: async () => ({ output_text: JSON.stringify(transportResponse) }),
|
||||||
|
});
|
||||||
|
const provider = createOpenAIReconstructionProvider({
|
||||||
|
apiKey: "test-key",
|
||||||
|
fetchImpl,
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await provider.generateReconstruction(
|
||||||
|
"prompt",
|
||||||
|
"test-model",
|
||||||
|
reconstructionGenerationJsonSchema,
|
||||||
|
{ outputZodSchema: reconstructionGenerationSchema },
|
||||||
|
);
|
||||||
|
const request = JSON.parse(fetchImpl.mock.calls[0][1].body);
|
||||||
|
const unexplainedSchema =
|
||||||
|
request.text.format.schema.properties.reconstruction.properties
|
||||||
|
.unexplainedTransitions.items;
|
||||||
|
|
||||||
|
expect(unexplainedSchema.required).toContain("entity");
|
||||||
|
expect(unexplainedSchema.properties.entity.anyOf).toContainEqual({ type: "null" });
|
||||||
|
expect(result.response.inputClassification).not.toHaveProperty("secondaryTypes");
|
||||||
|
expect(result.response.inputClassification).not.toHaveProperty("reasoningModes");
|
||||||
|
expect(result.response.reconstruction.unexplainedTransitions[0]).not.toHaveProperty("entity");
|
||||||
|
expect(result.response.reconstruction.unexplainedTransitions[0]).not.toHaveProperty("previousState");
|
||||||
|
expect(result.response.reconstruction.unexplainedTransitions[0]).not.toHaveProperty("currentState");
|
||||||
|
expect(result.response.reconstruction.plausibleInterpretations[0]).not.toHaveProperty("assumptionsRequired");
|
||||||
|
expect(result.response.reconstruction).not.toHaveProperty("relationships");
|
||||||
|
expect(result.response.evidence[0]).not.toHaveProperty("source");
|
||||||
|
expect(result.response.evidence[0].attribution).toBeNull();
|
||||||
|
expect(result.response.nextQuestion).not.toHaveProperty("reasoningMode");
|
||||||
|
expect(reconstructionGenerationSchema.safeParse(result.response).success).toBe(true);
|
||||||
|
expect(reconstructionV2Schema.safeParse(result.response).success).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
it("removes only optional transport null placeholders", () => {
|
it("removes only optional transport null placeholders", () => {
|
||||||
const schema = z.object({
|
const schema = z.object({
|
||||||
optionalText: z.string().optional(),
|
optionalText: z.string().optional(),
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import { normaliseAnalysisResponse } from "@/lib/reconstruction/compatibility.js";
|
import { normaliseAnalysisResponse } from "@/lib/reconstruction/compatibility.js";
|
||||||
|
import { reconstructionGenerationJsonSchema } from "@/lib/reconstruction/schema.js";
|
||||||
|
|
||||||
const mockGenerateReconstruction = vi.fn();
|
const mockGenerateReconstruction = vi.fn();
|
||||||
|
|
||||||
@@ -17,6 +18,7 @@ vi.mock("@/lib/llm/provider.js", () => ({
|
|||||||
getProvider: () => ({
|
getProvider: () => ({
|
||||||
generateReconstruction: (...args) => mockGenerateReconstruction(...args),
|
generateReconstruction: (...args) => mockGenerateReconstruction(...args),
|
||||||
}),
|
}),
|
||||||
|
getProviderModelName: () => "test-model",
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock("@/lib/reconstruction/prompt.js", () => ({
|
vi.mock("@/lib/reconstruction/prompt.js", () => ({
|
||||||
@@ -134,13 +136,117 @@ describe("analyseScenario compatibility", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
expect(result.success).toBe(true);
|
expect(result.success).toBe(true);
|
||||||
expect(injectedProvider.generateReconstruction).toHaveBeenCalledWith(
|
expect(injectedProvider.generateReconstruction).toHaveBeenCalledTimes(1);
|
||||||
"prompt",
|
const [prompt, modelName, outputSchema, options] =
|
||||||
"experiment-model",
|
injectedProvider.generateReconstruction.mock.calls[0];
|
||||||
);
|
expect(prompt).toBe("prompt");
|
||||||
|
expect(modelName).toBe("experiment-model");
|
||||||
|
expect(outputSchema).toEqual(reconstructionGenerationJsonSchema);
|
||||||
|
expect(options.outputZodSchema.safeParse).toBeTypeOf("function");
|
||||||
|
expect(
|
||||||
|
options.outputZodSchema.safeParse({
|
||||||
|
inputClassification: {
|
||||||
|
primaryType: "other",
|
||||||
|
secondaryTypes: [],
|
||||||
|
reasoningModes: [],
|
||||||
|
classificationReason: "reason",
|
||||||
|
confidence: "medium",
|
||||||
|
},
|
||||||
|
reconstruction: {
|
||||||
|
summary: "summary",
|
||||||
|
actors: [],
|
||||||
|
systemsOrObjects: [],
|
||||||
|
expectedStates: [],
|
||||||
|
observedStates: [],
|
||||||
|
differences: [],
|
||||||
|
knownTransitions: [],
|
||||||
|
unexplainedTransitions: [],
|
||||||
|
contradictions: [],
|
||||||
|
importantUnknowns: [],
|
||||||
|
plausibleInterpretations: [],
|
||||||
|
relationships: [],
|
||||||
|
},
|
||||||
|
evidence: [],
|
||||||
|
nextQuestion: {
|
||||||
|
id: "q1",
|
||||||
|
question: "What next?",
|
||||||
|
targets: [],
|
||||||
|
reason: "reason",
|
||||||
|
expectedInformationValue: "low",
|
||||||
|
reasoningMode: "other",
|
||||||
|
},
|
||||||
|
}).success,
|
||||||
|
).toBe(true);
|
||||||
expect(mockGenerateReconstruction).not.toHaveBeenCalled();
|
expect(mockGenerateReconstruction).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("preserves canonical reconstruction and reports invalid presentation nonfatally", async () => {
|
||||||
|
const response = {
|
||||||
|
inputClassification: {
|
||||||
|
primaryType: "other",
|
||||||
|
secondaryTypes: [],
|
||||||
|
reasoningModes: [],
|
||||||
|
classificationReason: "reason",
|
||||||
|
confidence: "medium",
|
||||||
|
},
|
||||||
|
reconstruction: {
|
||||||
|
summary: "summary",
|
||||||
|
actors: [],
|
||||||
|
systemsOrObjects: [],
|
||||||
|
expectedStates: [],
|
||||||
|
observedStates: [],
|
||||||
|
differences: [],
|
||||||
|
knownTransitions: [],
|
||||||
|
unexplainedTransitions: [],
|
||||||
|
contradictions: [],
|
||||||
|
importantUnknowns: [
|
||||||
|
{
|
||||||
|
id: "unknown-1",
|
||||||
|
description: "Whether workers can be recruited in time.",
|
||||||
|
question: "Whether workers can be recruited in time?",
|
||||||
|
confidence: "high",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
plausibleInterpretations: [],
|
||||||
|
relationships: [],
|
||||||
|
},
|
||||||
|
evidence: [],
|
||||||
|
nextQuestion: {
|
||||||
|
id: "q1",
|
||||||
|
question: "What recruitment evidence exists?",
|
||||||
|
targets: ["unknown-1"],
|
||||||
|
reason: "reason",
|
||||||
|
expectedInformationValue: "high",
|
||||||
|
reasoningMode: "other",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const injectedProvider = {
|
||||||
|
generateReconstruction: vi.fn().mockResolvedValue(response),
|
||||||
|
};
|
||||||
|
const { analyseScenario } = await import("@/lib/analysis.js");
|
||||||
|
|
||||||
|
const result = await analyseScenario("Scenario text", {
|
||||||
|
reconstructionProvider: injectedProvider,
|
||||||
|
reconstructionModelName: "experiment-model",
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result.success).toBe(true);
|
||||||
|
expect(result.validationStatus).toBe("valid");
|
||||||
|
expect(result.presentationValidationStatus).toBe("partial");
|
||||||
|
expect(result.reconstruction.importantUnknowns[0]).toEqual({
|
||||||
|
id: "unknown-1",
|
||||||
|
description: "Whether workers can be recruited in time.",
|
||||||
|
confidence: "high",
|
||||||
|
});
|
||||||
|
expect(result.presentationValidationIssues).toEqual([
|
||||||
|
expect.objectContaining({
|
||||||
|
category: "question_indirect_proposition",
|
||||||
|
unknownId: "unknown-1",
|
||||||
|
severity: "nonfatal",
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
it("preserves an attempted provider API path on provider failure", async () => {
|
it("preserves an attempted provider API path on provider failure", async () => {
|
||||||
const providerError = new Error("Provider failed");
|
const providerError = new Error("Provider failed");
|
||||||
providerError.providerApiPath = "/api/chat";
|
providerError.providerApiPath = "/api/chat";
|
||||||
|
|||||||
@@ -0,0 +1,142 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { promises as fs } from "node:fs";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import {
|
||||||
|
reconstructionGenerationJsonSchema,
|
||||||
|
reconstructionGenerationSchema,
|
||||||
|
reconstructionV2Schema,
|
||||||
|
} from "@/lib/reconstruction/schema.js";
|
||||||
|
import {
|
||||||
|
validateOpenQuestionCandidate,
|
||||||
|
validateReconstructionOpenQuestions,
|
||||||
|
} from "@/lib/reconstruction/open-question-presentation.js";
|
||||||
|
|
||||||
|
function makeAnalysisResponse(questionValue, includeQuestion = true) {
|
||||||
|
const unknown = {
|
||||||
|
id: "unknown-workforce",
|
||||||
|
description:
|
||||||
|
"Whether necessary skilled production workers can be recruited within the required project timeframe.",
|
||||||
|
confidence: "high",
|
||||||
|
};
|
||||||
|
if (includeQuestion) unknown.question = questionValue;
|
||||||
|
|
||||||
|
return {
|
||||||
|
inputClassification: {
|
||||||
|
primaryType: "other",
|
||||||
|
secondaryTypes: [],
|
||||||
|
reasoningModes: [],
|
||||||
|
classificationReason: "A delivery uncertainty is central to the situation.",
|
||||||
|
confidence: "high",
|
||||||
|
},
|
||||||
|
reconstruction: {
|
||||||
|
summary: "Delivery depends on recruiting skilled production workers.",
|
||||||
|
actors: [],
|
||||||
|
systemsOrObjects: [],
|
||||||
|
expectedStates: [],
|
||||||
|
observedStates: [],
|
||||||
|
differences: [],
|
||||||
|
knownTransitions: [],
|
||||||
|
unexplainedTransitions: [],
|
||||||
|
contradictions: [],
|
||||||
|
importantUnknowns: [unknown],
|
||||||
|
plausibleInterpretations: [],
|
||||||
|
relationships: [],
|
||||||
|
},
|
||||||
|
evidence: [],
|
||||||
|
nextQuestion: {
|
||||||
|
id: "next-1",
|
||||||
|
question: "What recruitment evidence is available?",
|
||||||
|
targets: ["unknown-workforce"],
|
||||||
|
reason: "Recruitment feasibility is unresolved.",
|
||||||
|
expectedInformationValue: "high",
|
||||||
|
reasoningMode: "other",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("new reconstruction generation contract", () => {
|
||||||
|
it("requires a non-empty question for every generated important unknown", () => {
|
||||||
|
expect(
|
||||||
|
reconstructionGenerationSchema.safeParse(
|
||||||
|
makeAnalysisResponse(undefined, false),
|
||||||
|
).success,
|
||||||
|
).toBe(false);
|
||||||
|
expect(
|
||||||
|
reconstructionGenerationSchema.safeParse(makeAnalysisResponse(" ")).success,
|
||||||
|
).toBe(false);
|
||||||
|
expect(
|
||||||
|
reconstructionGenerationSchema.safeParse(
|
||||||
|
makeAnalysisResponse(
|
||||||
|
"Can the necessary skilled production workers be recruited within the required project timeframe?",
|
||||||
|
),
|
||||||
|
).success,
|
||||||
|
).toBe(true);
|
||||||
|
|
||||||
|
const unknownSchema =
|
||||||
|
reconstructionGenerationJsonSchema.properties.reconstruction.properties
|
||||||
|
.importantUnknowns.items;
|
||||||
|
expect(unknownSchema.required).toContain("description");
|
||||||
|
expect(unknownSchema.required).toContain("question");
|
||||||
|
expect(unknownSchema.properties.question.minLength).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the durable compatibility schema tolerant of a missing question", () => {
|
||||||
|
expect(
|
||||||
|
reconstructionV2Schema.safeParse(makeAnalysisResponse(undefined, false)).success,
|
||||||
|
).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Open Question presentation validation", () => {
|
||||||
|
it("accepts a trimmed direct interrogative", () => {
|
||||||
|
expect(
|
||||||
|
validateOpenQuestionCandidate(
|
||||||
|
" Can the necessary workers be recruited in time? ",
|
||||||
|
),
|
||||||
|
).toEqual({
|
||||||
|
valid: true,
|
||||||
|
question: "Can the necessary workers be recruited in time?",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses canonical UI fallback without a technical explanation", async () => {
|
||||||
|
const source = await fs.readFile(
|
||||||
|
join(process.cwd(), "components/reasoning-workspace.jsx"),
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(source).toContain("const openQuestion = node.openQuestion || node.label;");
|
||||||
|
expect(source).not.toContain(
|
||||||
|
"Canonical uncertainty — direct question wording unavailable.",
|
||||||
|
);
|
||||||
|
expect(source).toContain("startFocused(n.id);");
|
||||||
|
});
|
||||||
|
|
||||||
|
it.each([
|
||||||
|
["question_missing", undefined, false],
|
||||||
|
["question_empty", " ", true],
|
||||||
|
["question_indirect_proposition", "Whether workers can be recruited in time", true],
|
||||||
|
["question_indirect_proposition", "Whether workers can be recruited in time?", true],
|
||||||
|
["question_not_interrogative", "Workers can be recruited in time.", true],
|
||||||
|
])("classifies %s nonfatally", (category, candidate, includeQuestion) => {
|
||||||
|
const input = makeAnalysisResponse(candidate, includeQuestion).reconstruction;
|
||||||
|
const canonicalBefore = input.importantUnknowns[0].description;
|
||||||
|
const idBefore = input.importantUnknowns[0].id;
|
||||||
|
|
||||||
|
const result = validateReconstructionOpenQuestions(input);
|
||||||
|
|
||||||
|
expect(result.reconstruction.importantUnknowns[0]).not.toHaveProperty("question");
|
||||||
|
expect(result.reconstruction.importantUnknowns[0].description).toBe(
|
||||||
|
canonicalBefore,
|
||||||
|
);
|
||||||
|
expect(result.reconstruction.importantUnknowns[0].id).toBe(idBefore);
|
||||||
|
expect(result.issues).toEqual([
|
||||||
|
expect.objectContaining({
|
||||||
|
category,
|
||||||
|
unknownId: idBefore,
|
||||||
|
severity: "nonfatal",
|
||||||
|
path: ["reconstruction", "importantUnknowns", 0, "question"],
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user