wip(confidence-engine): checkpoint open-question presentation work
This commit is contained in:
@@ -7,11 +7,13 @@ function mkN(id, label, opts) {
|
||||
var k = (opts && opts.kind) || "unknown";
|
||||
var s = (opts && opts.status) || (k === "unknown" ? "unknown" : "known");
|
||||
var c = (opts && opts.confidence) || "low";
|
||||
return {
|
||||
var node = {
|
||||
id, label, description: (opts && opts.description) || label, kind: k, status: s, confidence: c,
|
||||
confidenceAssessment: { evidenceConfidence: c, completenessStatus: "partial", conclusionConfidence: c },
|
||||
value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], childIds: []
|
||||
};
|
||||
if (opts && opts.openQuestion) node.openQuestion = opts.openQuestion;
|
||||
return node;
|
||||
}
|
||||
|
||||
/* ── Filtering tests ─────────────────────────────────────────────── */
|
||||
@@ -147,6 +149,41 @@ describe("Deduplication", () => {
|
||||
});
|
||||
// The unresolved one (if any) or the cross-dedup should prevent duplication
|
||||
});
|
||||
|
||||
it("keeps canonically distinct uncertainties with identical presentation questions", () => {
|
||||
const sharedQuestion = "What evidence would resolve this uncertainty?";
|
||||
const vm = buildFacilitatorViewModel({
|
||||
nodes: [
|
||||
mkN("u1", "Whether supplier capacity is sufficient for launch", {
|
||||
openQuestion: sharedQuestion,
|
||||
}),
|
||||
mkN("u2", "Whether customer demand is sufficient for launch", {
|
||||
openQuestion: sharedQuestion,
|
||||
}),
|
||||
],
|
||||
resolvedIds: new Set(),
|
||||
});
|
||||
|
||||
expect(vm.investigating.items.filter((item) => item === sharedQuestion)).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("continues to merge genuine canonical duplicates despite different presentation questions", () => {
|
||||
const vm = buildFacilitatorViewModel({
|
||||
nodes: [
|
||||
mkN("u1", "Whether supplier capacity is sufficient for launch", {
|
||||
openQuestion: "Can the supplier support the launch volume?",
|
||||
}),
|
||||
mkN("u2", "whether supplier capacity is sufficient for launch!", {
|
||||
openQuestion: "Is enough supplier capacity available?",
|
||||
}),
|
||||
],
|
||||
resolvedIds: new Set(),
|
||||
});
|
||||
|
||||
expect(vm.investigating.items).toEqual([
|
||||
"Can the supplier support the launch volume?",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
/* ── Ranking tests ───────────────────────────────────────────────── */
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildInitialGraph } from "@/lib/graph/builder.js";
|
||||
import { formulateQuestionForTarget } from "@/lib/graph/focused-investigation.js";
|
||||
import { makeGraph, makeNodeId } from "@/lib/graph/schema.js";
|
||||
import { buildFacilitatorViewModel } from "@/lib/presentation/facilitator-view-adapter.js";
|
||||
import { buildPrompt } from "@/lib/reconstruction/prompt.js";
|
||||
import { validateReconstructionOpenQuestions } from "@/lib/reconstruction/open-question-presentation.js";
|
||||
import { reconstructionV2Schema } from "@/lib/reconstruction/schema.js";
|
||||
|
||||
function reconstructionWithUnknowns(importantUnknowns) {
|
||||
return {
|
||||
inputClassification: {
|
||||
primaryType: "other",
|
||||
secondaryTypes: [],
|
||||
reasoningModes: ["identify_missing_information"],
|
||||
classificationReason: "The scenario contains unresolved factors.",
|
||||
confidence: "medium",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: "A situation with unresolved factors.",
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [],
|
||||
differences: [],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: [],
|
||||
importantUnknowns,
|
||||
plausibleInterpretations: [],
|
||||
relationships: [],
|
||||
},
|
||||
evidence: [],
|
||||
nextQuestion: {
|
||||
id: "next-question",
|
||||
question: "What information would clarify the situation?",
|
||||
targets: [],
|
||||
reason: "The situation has unresolved factors.",
|
||||
expectedInformationValue: "medium",
|
||||
reasoningMode: "identify_missing_information",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const CASES = [
|
||||
{
|
||||
name: "recruitment uncertainty",
|
||||
description: "Whether skilled production staff can be recruited within the required timeframe",
|
||||
question: "Can skilled production staff be recruited within the required timeframe?",
|
||||
terms: ["skilled production staff", "required timeframe"],
|
||||
},
|
||||
{
|
||||
name: "demand uncertainty",
|
||||
description: "The level of demand in the North West market needed to justify the expansion cost",
|
||||
question: "What level of demand in the North West market is needed to justify the expansion cost?",
|
||||
terms: ["North West market", "justify the expansion cost"],
|
||||
},
|
||||
{
|
||||
name: "historical uncertainty",
|
||||
description: "Whether the prior staffing level was sufficient before the expansion",
|
||||
question: "Was the prior staffing level sufficient before the expansion?",
|
||||
terms: ["prior staffing level", "before the expansion"],
|
||||
},
|
||||
{
|
||||
name: "non-business uncertainty",
|
||||
description: "The time at which the missing hiker last used the eastern trail",
|
||||
question: "When did the missing hiker last use the eastern trail?",
|
||||
terms: ["missing hiker", "eastern trail"],
|
||||
},
|
||||
];
|
||||
|
||||
describe("Open Question presentation", () => {
|
||||
it.each(["v0.2", "v0.3", "v0.4", "v0.5"])("instructs reconstruction prompt %s to produce a direct Open Question", async (version) => {
|
||||
const { prompt } = await buildPrompt("A scenario with an unresolved condition.", version);
|
||||
|
||||
expect(prompt).toContain("importantUnknown");
|
||||
expect(prompt).toContain("natural, direct interrogative");
|
||||
expect(prompt).toContain('"question"');
|
||||
expect(prompt).toContain('"Whether …"');
|
||||
});
|
||||
|
||||
it.each(CASES)("preserves $name meaning while displaying a direct interrogative", ({ description, question, terms }) => {
|
||||
const parsed = reconstructionV2Schema.parse(
|
||||
reconstructionWithUnknowns([{ id: "unknown", description, question, confidence: "medium" }]),
|
||||
);
|
||||
const graph = buildInitialGraph({ reconstruction: parsed.reconstruction, evidence: [] });
|
||||
const unknown = graph.nodes.find((node) => node.kind === "unknown");
|
||||
const viewModel = buildFacilitatorViewModel({ nodes: graph.nodes, resolvedIds: new Set() });
|
||||
|
||||
expect(unknown.label).toBe(description);
|
||||
expect(unknown.description).toBe(description);
|
||||
expect(unknown.openQuestion).toBe(question);
|
||||
expect(viewModel.investigating.items).toContain(question);
|
||||
expect(question).toMatch(/\?$/);
|
||||
expect(question).not.toMatch(/^whether\b/i);
|
||||
expect(question).not.toMatch(/^before whether\b/i);
|
||||
for (const term of terms) expect(question.toLowerCase()).toContain(term.toLowerCase());
|
||||
});
|
||||
|
||||
it("leaves canonical identity, explicit target selection, and focused formulation unchanged", () => {
|
||||
const description = CASES[0].description;
|
||||
const question = CASES[0].question;
|
||||
const graphData = buildInitialGraph({
|
||||
reconstruction: reconstructionWithUnknowns([
|
||||
{ id: "recruitment", description, question, confidence: "medium" },
|
||||
]).reconstruction,
|
||||
evidence: [],
|
||||
});
|
||||
const target = graphData.nodes.find((node) => node.kind === "unknown");
|
||||
const graph = makeGraph({
|
||||
centralStatement: "A manufacturer is considering expansion.",
|
||||
nodes: graphData.nodes,
|
||||
edges: graphData.edges,
|
||||
activeUnknownNodeId: null,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Expansion depends on recruitment capacity.",
|
||||
});
|
||||
const before = JSON.stringify(graph);
|
||||
const result = formulateQuestionForTarget({ situationGraph: graph, targetNodeId: target.id });
|
||||
|
||||
expect(target.id).toBe(makeNodeId(description));
|
||||
expect(target.id).not.toBe(makeNodeId(question));
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.targetNodeId).toBe(target.id);
|
||||
expect(result.question).toMatch(/\?$/);
|
||||
expect(result.question.toLowerCase()).toContain("staff");
|
||||
expect(JSON.stringify(graph)).toBe(before);
|
||||
});
|
||||
|
||||
it("keeps persisted legacy unknowns valid and falls back to canonical text when no display question exists", () => {
|
||||
const legacyDescription = "Whether the figures are comparable";
|
||||
const legacyNode = makeGraph({
|
||||
centralStatement: "Compare two figures.",
|
||||
nodes: [
|
||||
{
|
||||
id: makeNodeId(legacyDescription),
|
||||
label: legacyDescription,
|
||||
description: legacyDescription,
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
},
|
||||
],
|
||||
edges: [],
|
||||
activeUnknownNodeId: null,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "The comparison remains unresolved.",
|
||||
});
|
||||
const viewModel = buildFacilitatorViewModel({ nodes: legacyNode.nodes, resolvedIds: new Set() });
|
||||
|
||||
expect(viewModel.investigating.items).toContain(legacyDescription);
|
||||
expect(legacyNode.nodes[0]).not.toHaveProperty("openQuestion");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["missing", undefined, false],
|
||||
["empty", " ", true],
|
||||
["whether fragment", "Whether the facility can recruit workers in time", true],
|
||||
["whether fragment with punctuation", "Whether the facility can recruit workers in time?", true],
|
||||
["statement", "The facility can recruit workers in time.", true],
|
||||
])(
|
||||
"keeps an invalid %s presentation canonically selectable and investigable",
|
||||
(_name, question, includeQuestion) => {
|
||||
const description =
|
||||
"Whether the facility can recruit skilled workers within the launch timeframe.";
|
||||
const unknown = {
|
||||
id: "unknown",
|
||||
description,
|
||||
confidence: "medium",
|
||||
};
|
||||
if (includeQuestion) unknown.question = question;
|
||||
const reconstruction = reconstructionWithUnknowns([unknown]).reconstruction;
|
||||
const validated = validateReconstructionOpenQuestions(reconstruction);
|
||||
const graphParts = buildInitialGraph({
|
||||
reconstruction: validated.reconstruction,
|
||||
evidence: [],
|
||||
});
|
||||
const graph = makeGraph({
|
||||
centralStatement: "The facility launch depends on workforce availability.",
|
||||
nodes: graphParts.nodes,
|
||||
edges: graphParts.edges,
|
||||
currentSummary: "Workforce availability remains unresolved.",
|
||||
});
|
||||
const graphUnknown = graph.nodes.find((node) => node.kind === "unknown");
|
||||
const focused = formulateQuestionForTarget({
|
||||
situationGraph: graph,
|
||||
targetNodeId: graphUnknown.id,
|
||||
});
|
||||
|
||||
expect(validated.issues).toHaveLength(1);
|
||||
expect(graphUnknown.label).toBe(description);
|
||||
expect(graphUnknown.description).toBe(description);
|
||||
expect(graphUnknown).not.toHaveProperty("openQuestion");
|
||||
expect(graphUnknown.id).toBe(makeNodeId(description));
|
||||
expect(focused.success).toBe(true);
|
||||
expect(focused.targetNodeId).toBe(graphUnknown.id);
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -55,6 +55,8 @@ function makeAnalysisResult(overrides = {}) {
|
||||
compatibilityApplied: false,
|
||||
compatibilityChanges: [],
|
||||
compatibilityWarnings: [],
|
||||
presentationValidationStatus: "valid",
|
||||
presentationValidationIssues: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
@@ -399,9 +401,37 @@ describe("lib/graph/orchestrator startCase", () => {
|
||||
valid: true,
|
||||
errors: [],
|
||||
});
|
||||
expect(result.diagnostics.presentationValidationStatus).toBe("valid");
|
||||
expect(result.diagnostics.presentationValidationIssues).toEqual([]);
|
||||
expect(downstreamProvider.generateReconstruction).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("propagates nonfatal Open Question presentation issues through diagnostics", async () => {
|
||||
const issue = {
|
||||
category: "question_indirect_proposition",
|
||||
path: ["reconstruction", "importantUnknowns", 0, "question"],
|
||||
unknownId: "unk-1",
|
||||
message: "The generated Open Question used an indirect uncertainty proposition.",
|
||||
severity: "nonfatal",
|
||||
};
|
||||
mockAnalyseScenario.mockResolvedValue(
|
||||
makeAnalysisResult({
|
||||
presentationValidationStatus: "partial",
|
||||
presentationValidationIssues: [issue],
|
||||
}),
|
||||
);
|
||||
const { startCase } = await import("@/lib/graph/orchestrator.js");
|
||||
|
||||
const result = await startCase(
|
||||
{ scenario: "Scenario text" },
|
||||
{ reconstructionOnly: true },
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.diagnostics.presentationValidationStatus).toBe("partial");
|
||||
expect(result.diagnostics.presentationValidationIssues).toEqual([issue]);
|
||||
});
|
||||
|
||||
it("rejects invalid request input without throwing", async () => {
|
||||
const { startCase } = await import("@/lib/graph/orchestrator.js");
|
||||
|
||||
|
||||
+136
-1
@@ -8,7 +8,11 @@ import {
|
||||
normaliseOpenAITransportResponse,
|
||||
reconstructionJsonSchema,
|
||||
} from "@/lib/llm/provider.js";
|
||||
import { reconstructionV2Schema } from "@/lib/reconstruction/schema.js";
|
||||
import {
|
||||
reconstructionGenerationJsonSchema,
|
||||
reconstructionGenerationSchema,
|
||||
reconstructionV2Schema,
|
||||
} from "@/lib/reconstruction/schema.js";
|
||||
import { focusedDeconstructJsonSchema } from "@/lib/graph/focused-investigation.js";
|
||||
import { z } from "zod";
|
||||
|
||||
@@ -299,6 +303,137 @@ describe("OpenAI reconstruction provider experiment seam", () => {
|
||||
expect(acceptsNull(transition.properties.id)).toBe(false);
|
||||
});
|
||||
|
||||
it("preserves optional transport nullability for the generation-specific schema", async () => {
|
||||
const fetchImpl = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ output_text: "{}" }),
|
||||
});
|
||||
const provider = createOpenAIReconstructionProvider({
|
||||
apiKey: "test-key",
|
||||
fetchImpl,
|
||||
});
|
||||
|
||||
await provider.generateReconstruction(
|
||||
"prompt",
|
||||
"test-model",
|
||||
reconstructionGenerationJsonSchema,
|
||||
{ outputZodSchema: reconstructionGenerationSchema },
|
||||
);
|
||||
|
||||
const request = JSON.parse(fetchImpl.mock.calls[0][1].body);
|
||||
const reconstruction = request.text.format.schema.properties.reconstruction;
|
||||
const unknown = reconstruction.properties.importantUnknowns.items;
|
||||
const unexplained = reconstruction.properties.unexplainedTransitions.items;
|
||||
|
||||
expect(unknown.required).toContain("question");
|
||||
expect(unknown.properties.question.minLength).toBe(1);
|
||||
expect(unexplained.required).toContain("entity");
|
||||
expect(unexplained.properties.entity.anyOf).toContainEqual({ type: "null" });
|
||||
});
|
||||
|
||||
it("normalises generation-schema null placeholders before canonical validation", async () => {
|
||||
const transportResponse = {
|
||||
inputClassification: {
|
||||
primaryType: "unexplained_change",
|
||||
secondaryTypes: null,
|
||||
reasoningModes: null,
|
||||
classificationReason: "A reported change requires reconstruction.",
|
||||
confidence: "high",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: "A change occurred and its cause remains uncertain.",
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [],
|
||||
differences: [],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [
|
||||
{
|
||||
id: "transition-1",
|
||||
description: "The service level declined after the release.",
|
||||
confidence: "medium",
|
||||
entity: null,
|
||||
previousState: null,
|
||||
currentState: null,
|
||||
},
|
||||
],
|
||||
contradictions: [],
|
||||
importantUnknowns: [
|
||||
{
|
||||
id: "unknown-1",
|
||||
description: "Whether the release caused the service-level decline",
|
||||
question: "Did the release cause the service-level decline?",
|
||||
confidence: "medium",
|
||||
},
|
||||
],
|
||||
plausibleInterpretations: [
|
||||
{
|
||||
id: "interpretation-1",
|
||||
description: "The release may have reduced service capacity.",
|
||||
supportingEvidenceIds: ["evidence-1"],
|
||||
assumptionsRequired: null,
|
||||
confidence: "low",
|
||||
},
|
||||
],
|
||||
relationships: null,
|
||||
},
|
||||
evidence: [
|
||||
{
|
||||
id: "evidence-1",
|
||||
description: "The decline was reported after the release.",
|
||||
evidenceType: "reported_statement",
|
||||
source: null,
|
||||
attribution: null,
|
||||
confidence: "medium",
|
||||
importance: "important",
|
||||
},
|
||||
],
|
||||
nextQuestion: {
|
||||
id: "next-1",
|
||||
question: "What changed in service capacity after the release?",
|
||||
targets: ["unknown-1"],
|
||||
reason: "The cause of the decline remains unresolved.",
|
||||
expectedInformationValue: "high",
|
||||
reasoningMode: null,
|
||||
},
|
||||
};
|
||||
const fetchImpl = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ output_text: JSON.stringify(transportResponse) }),
|
||||
});
|
||||
const provider = createOpenAIReconstructionProvider({
|
||||
apiKey: "test-key",
|
||||
fetchImpl,
|
||||
});
|
||||
|
||||
const result = await provider.generateReconstruction(
|
||||
"prompt",
|
||||
"test-model",
|
||||
reconstructionGenerationJsonSchema,
|
||||
{ outputZodSchema: reconstructionGenerationSchema },
|
||||
);
|
||||
const request = JSON.parse(fetchImpl.mock.calls[0][1].body);
|
||||
const unexplainedSchema =
|
||||
request.text.format.schema.properties.reconstruction.properties
|
||||
.unexplainedTransitions.items;
|
||||
|
||||
expect(unexplainedSchema.required).toContain("entity");
|
||||
expect(unexplainedSchema.properties.entity.anyOf).toContainEqual({ type: "null" });
|
||||
expect(result.response.inputClassification).not.toHaveProperty("secondaryTypes");
|
||||
expect(result.response.inputClassification).not.toHaveProperty("reasoningModes");
|
||||
expect(result.response.reconstruction.unexplainedTransitions[0]).not.toHaveProperty("entity");
|
||||
expect(result.response.reconstruction.unexplainedTransitions[0]).not.toHaveProperty("previousState");
|
||||
expect(result.response.reconstruction.unexplainedTransitions[0]).not.toHaveProperty("currentState");
|
||||
expect(result.response.reconstruction.plausibleInterpretations[0]).not.toHaveProperty("assumptionsRequired");
|
||||
expect(result.response.reconstruction).not.toHaveProperty("relationships");
|
||||
expect(result.response.evidence[0]).not.toHaveProperty("source");
|
||||
expect(result.response.evidence[0].attribution).toBeNull();
|
||||
expect(result.response.nextQuestion).not.toHaveProperty("reasoningMode");
|
||||
expect(reconstructionGenerationSchema.safeParse(result.response).success).toBe(true);
|
||||
expect(reconstructionV2Schema.safeParse(result.response).success).toBe(true);
|
||||
});
|
||||
|
||||
it("removes only optional transport null placeholders", () => {
|
||||
const schema = z.object({
|
||||
optionalText: z.string().optional(),
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { normaliseAnalysisResponse } from "@/lib/reconstruction/compatibility.js";
|
||||
import { reconstructionGenerationJsonSchema } from "@/lib/reconstruction/schema.js";
|
||||
|
||||
const mockGenerateReconstruction = vi.fn();
|
||||
|
||||
@@ -17,6 +18,7 @@ vi.mock("@/lib/llm/provider.js", () => ({
|
||||
getProvider: () => ({
|
||||
generateReconstruction: (...args) => mockGenerateReconstruction(...args),
|
||||
}),
|
||||
getProviderModelName: () => "test-model",
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/reconstruction/prompt.js", () => ({
|
||||
@@ -134,13 +136,117 @@ describe("analyseScenario compatibility", () => {
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(injectedProvider.generateReconstruction).toHaveBeenCalledWith(
|
||||
"prompt",
|
||||
"experiment-model",
|
||||
);
|
||||
expect(injectedProvider.generateReconstruction).toHaveBeenCalledTimes(1);
|
||||
const [prompt, modelName, outputSchema, options] =
|
||||
injectedProvider.generateReconstruction.mock.calls[0];
|
||||
expect(prompt).toBe("prompt");
|
||||
expect(modelName).toBe("experiment-model");
|
||||
expect(outputSchema).toEqual(reconstructionGenerationJsonSchema);
|
||||
expect(options.outputZodSchema.safeParse).toBeTypeOf("function");
|
||||
expect(
|
||||
options.outputZodSchema.safeParse({
|
||||
inputClassification: {
|
||||
primaryType: "other",
|
||||
secondaryTypes: [],
|
||||
reasoningModes: [],
|
||||
classificationReason: "reason",
|
||||
confidence: "medium",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: "summary",
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [],
|
||||
differences: [],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: [],
|
||||
importantUnknowns: [],
|
||||
plausibleInterpretations: [],
|
||||
relationships: [],
|
||||
},
|
||||
evidence: [],
|
||||
nextQuestion: {
|
||||
id: "q1",
|
||||
question: "What next?",
|
||||
targets: [],
|
||||
reason: "reason",
|
||||
expectedInformationValue: "low",
|
||||
reasoningMode: "other",
|
||||
},
|
||||
}).success,
|
||||
).toBe(true);
|
||||
expect(mockGenerateReconstruction).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("preserves canonical reconstruction and reports invalid presentation nonfatally", async () => {
|
||||
const response = {
|
||||
inputClassification: {
|
||||
primaryType: "other",
|
||||
secondaryTypes: [],
|
||||
reasoningModes: [],
|
||||
classificationReason: "reason",
|
||||
confidence: "medium",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: "summary",
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [],
|
||||
differences: [],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: [],
|
||||
importantUnknowns: [
|
||||
{
|
||||
id: "unknown-1",
|
||||
description: "Whether workers can be recruited in time.",
|
||||
question: "Whether workers can be recruited in time?",
|
||||
confidence: "high",
|
||||
},
|
||||
],
|
||||
plausibleInterpretations: [],
|
||||
relationships: [],
|
||||
},
|
||||
evidence: [],
|
||||
nextQuestion: {
|
||||
id: "q1",
|
||||
question: "What recruitment evidence exists?",
|
||||
targets: ["unknown-1"],
|
||||
reason: "reason",
|
||||
expectedInformationValue: "high",
|
||||
reasoningMode: "other",
|
||||
},
|
||||
};
|
||||
const injectedProvider = {
|
||||
generateReconstruction: vi.fn().mockResolvedValue(response),
|
||||
};
|
||||
const { analyseScenario } = await import("@/lib/analysis.js");
|
||||
|
||||
const result = await analyseScenario("Scenario text", {
|
||||
reconstructionProvider: injectedProvider,
|
||||
reconstructionModelName: "experiment-model",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.validationStatus).toBe("valid");
|
||||
expect(result.presentationValidationStatus).toBe("partial");
|
||||
expect(result.reconstruction.importantUnknowns[0]).toEqual({
|
||||
id: "unknown-1",
|
||||
description: "Whether workers can be recruited in time.",
|
||||
confidence: "high",
|
||||
});
|
||||
expect(result.presentationValidationIssues).toEqual([
|
||||
expect.objectContaining({
|
||||
category: "question_indirect_proposition",
|
||||
unknownId: "unknown-1",
|
||||
severity: "nonfatal",
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves an attempted provider API path on provider failure", async () => {
|
||||
const providerError = new Error("Provider failed");
|
||||
providerError.providerApiPath = "/api/chat";
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { promises as fs } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
reconstructionGenerationJsonSchema,
|
||||
reconstructionGenerationSchema,
|
||||
reconstructionV2Schema,
|
||||
} from "@/lib/reconstruction/schema.js";
|
||||
import {
|
||||
validateOpenQuestionCandidate,
|
||||
validateReconstructionOpenQuestions,
|
||||
} from "@/lib/reconstruction/open-question-presentation.js";
|
||||
|
||||
function makeAnalysisResponse(questionValue, includeQuestion = true) {
|
||||
const unknown = {
|
||||
id: "unknown-workforce",
|
||||
description:
|
||||
"Whether necessary skilled production workers can be recruited within the required project timeframe.",
|
||||
confidence: "high",
|
||||
};
|
||||
if (includeQuestion) unknown.question = questionValue;
|
||||
|
||||
return {
|
||||
inputClassification: {
|
||||
primaryType: "other",
|
||||
secondaryTypes: [],
|
||||
reasoningModes: [],
|
||||
classificationReason: "A delivery uncertainty is central to the situation.",
|
||||
confidence: "high",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: "Delivery depends on recruiting skilled production workers.",
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [],
|
||||
differences: [],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: [],
|
||||
importantUnknowns: [unknown],
|
||||
plausibleInterpretations: [],
|
||||
relationships: [],
|
||||
},
|
||||
evidence: [],
|
||||
nextQuestion: {
|
||||
id: "next-1",
|
||||
question: "What recruitment evidence is available?",
|
||||
targets: ["unknown-workforce"],
|
||||
reason: "Recruitment feasibility is unresolved.",
|
||||
expectedInformationValue: "high",
|
||||
reasoningMode: "other",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("new reconstruction generation contract", () => {
|
||||
it("requires a non-empty question for every generated important unknown", () => {
|
||||
expect(
|
||||
reconstructionGenerationSchema.safeParse(
|
||||
makeAnalysisResponse(undefined, false),
|
||||
).success,
|
||||
).toBe(false);
|
||||
expect(
|
||||
reconstructionGenerationSchema.safeParse(makeAnalysisResponse(" ")).success,
|
||||
).toBe(false);
|
||||
expect(
|
||||
reconstructionGenerationSchema.safeParse(
|
||||
makeAnalysisResponse(
|
||||
"Can the necessary skilled production workers be recruited within the required project timeframe?",
|
||||
),
|
||||
).success,
|
||||
).toBe(true);
|
||||
|
||||
const unknownSchema =
|
||||
reconstructionGenerationJsonSchema.properties.reconstruction.properties
|
||||
.importantUnknowns.items;
|
||||
expect(unknownSchema.required).toContain("description");
|
||||
expect(unknownSchema.required).toContain("question");
|
||||
expect(unknownSchema.properties.question.minLength).toBe(1);
|
||||
});
|
||||
|
||||
it("keeps the durable compatibility schema tolerant of a missing question", () => {
|
||||
expect(
|
||||
reconstructionV2Schema.safeParse(makeAnalysisResponse(undefined, false)).success,
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Open Question presentation validation", () => {
|
||||
it("accepts a trimmed direct interrogative", () => {
|
||||
expect(
|
||||
validateOpenQuestionCandidate(
|
||||
" Can the necessary workers be recruited in time? ",
|
||||
),
|
||||
).toEqual({
|
||||
valid: true,
|
||||
question: "Can the necessary workers be recruited in time?",
|
||||
});
|
||||
});
|
||||
|
||||
it("uses canonical UI fallback without a technical explanation", async () => {
|
||||
const source = await fs.readFile(
|
||||
join(process.cwd(), "components/reasoning-workspace.jsx"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
expect(source).toContain("const openQuestion = node.openQuestion || node.label;");
|
||||
expect(source).not.toContain(
|
||||
"Canonical uncertainty — direct question wording unavailable.",
|
||||
);
|
||||
expect(source).toContain("startFocused(n.id);");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["question_missing", undefined, false],
|
||||
["question_empty", " ", true],
|
||||
["question_indirect_proposition", "Whether workers can be recruited in time", true],
|
||||
["question_indirect_proposition", "Whether workers can be recruited in time?", true],
|
||||
["question_not_interrogative", "Workers can be recruited in time.", true],
|
||||
])("classifies %s nonfatally", (category, candidate, includeQuestion) => {
|
||||
const input = makeAnalysisResponse(candidate, includeQuestion).reconstruction;
|
||||
const canonicalBefore = input.importantUnknowns[0].description;
|
||||
const idBefore = input.importantUnknowns[0].id;
|
||||
|
||||
const result = validateReconstructionOpenQuestions(input);
|
||||
|
||||
expect(result.reconstruction.importantUnknowns[0]).not.toHaveProperty("question");
|
||||
expect(result.reconstruction.importantUnknowns[0].description).toBe(
|
||||
canonicalBefore,
|
||||
);
|
||||
expect(result.reconstruction.importantUnknowns[0].id).toBe(idBefore);
|
||||
expect(result.issues).toEqual([
|
||||
expect.objectContaining({
|
||||
category,
|
||||
unknownId: idBefore,
|
||||
severity: "nonfatal",
|
||||
path: ["reconstruction", "importantUnknowns", 0, "question"],
|
||||
}),
|
||||
]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user